You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/11/04 01:58:47 UTC
[2/5] systemml git commit: [MINOR] Performance createvar,
rmvar and cleanup meta data handling
[MINOR] Performance createvar, rmvar and cleanup meta data handling
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/d7d312c3
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/d7d312c3
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/d7d312c3
Branch: refs/heads/master
Commit: d7d312c39b5c6c90ca6bc487e6198eabf229dfc1
Parents: f76f213
Author: Matthias Boehm <mb...@gmail.com>
Authored: Thu Nov 2 22:59:23 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Fri Nov 3 18:59:28 2017 -0700
----------------------------------------------------------------------
.../apache/sysml/api/jmlc/PreparedScript.java | 6 +-
.../apache/sysml/api/mlcontext/MLContext.java | 6 +-
.../api/mlcontext/MLContextConversionUtil.java | 20 ++---
.../apache/sysml/hops/cost/CostEstimator.java | 3 +-
.../sysml/hops/ipa/InterProceduralAnalysis.java | 6 +-
.../apache/sysml/hops/recompile/Recompiler.java | 10 +--
.../RewriteRemovePersistentReadWrite.java | 6 +-
.../apache/sysml/lops/runtime/RunMRJobs.java | 7 +-
.../ExternalFunctionProgramBlock.java | 4 +-
.../ExternalFunctionProgramBlockCP.java | 4 +-
.../controlprogram/caching/CacheableData.java | 20 +++--
.../controlprogram/caching/FrameObject.java | 11 ++-
.../controlprogram/caching/MatrixObject.java | 58 +++++++--------
.../context/ExecutionContext.java | 46 +++++-------
.../controlprogram/parfor/DataPartitioner.java | 6 +-
.../controlprogram/parfor/ProgramConverter.java | 6 +-
.../parfor/ResultMergeLocalFile.java | 22 +++---
.../parfor/ResultMergeLocalMemory.java | 6 +-
.../parfor/ResultMergeRemoteMR.java | 6 +-
.../parfor/ResultMergeRemoteSpark.java | 6 +-
.../parfor/opt/OptimizerRuleBased.java | 4 +-
.../runtime/instructions/MRJobInstruction.java | 15 ++--
.../cp/DataPartitionCPInstruction.java | 4 +-
.../cp/QuantilePickCPInstruction.java | 12 +--
.../instructions/cp/VariableCPInstruction.java | 21 ++----
.../cpfile/MatrixIndexingCPFileInstruction.java | 6 +-
.../ParameterizedBuiltinCPFileInstruction.java | 8 +-
.../spark/CSVReblockSPInstruction.java | 4 +-
.../spark/ConvolutionSPInstruction.java | 4 +-
.../spark/ReblockSPInstruction.java | 4 +-
.../org/apache/sysml/runtime/matrix/GMR.java | 9 +--
.../apache/sysml/runtime/matrix/JobReturn.java | 27 ++-----
.../runtime/matrix/MatrixCharacteristics.java | 27 +++----
.../matrix/MatrixDimensionsMetaData.java | 78 --------------------
.../runtime/matrix/MatrixFormatMetaData.java | 53 -------------
.../apache/sysml/runtime/matrix/MetaData.java | 40 ++++++----
.../sysml/runtime/matrix/MetaDataFormat.java | 49 ++++++++++++
.../matrix/MetaDataNumItemsByEachReducer.java | 53 +++++++++++++
.../data/NumItemsByEachReducerMetaData.java | 64 ----------------
.../matrix/sort/PickFromCompactInputFormat.java | 8 +-
.../sysml/runtime/util/MapReduceTool.java | 8 +-
.../sysml/runtime/util/UtilFunctions.java | 6 +-
src/main/java/org/apache/sysml/udf/Matrix.java | 4 +-
.../sysml/yarn/ropt/ResourceOptimizer.java | 5 +-
.../functions/frame/FrameEvictionTest.java | 4 +-
45 files changed, 327 insertions(+), 449 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java b/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
index f14d3a1..c712bb2 100644
--- a/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
+++ b/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
@@ -49,7 +49,7 @@ import org.apache.sysml.runtime.instructions.cp.IntObject;
import org.apache.sysml.runtime.instructions.cp.ScalarObject;
import org.apache.sysml.runtime.instructions.cp.StringObject;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
@@ -258,7 +258,7 @@ public class PreparedScript
//create new matrix object
MatrixCharacteristics mc = new MatrixCharacteristics(matrix.getNumRows(), matrix.getNumColumns(), blocksize, blocksize);
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
+ MetaDataFormat meta = new MetaDataFormat(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
MatrixObject mo = new MatrixObject(ValueType.DOUBLE, OptimizerUtils.getUniqueTempFileName(), meta);
mo.acquireModify(matrix);
mo.release();
@@ -365,7 +365,7 @@ public class PreparedScript
//create new frame object
MatrixCharacteristics mc = new MatrixCharacteristics(frame.getNumRows(), frame.getNumColumns(), -1, -1);
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc, OutputInfo.BinaryCellOutputInfo, InputInfo.BinaryCellInputInfo);
+ MetaDataFormat meta = new MetaDataFormat(mc, OutputInfo.BinaryCellOutputInfo, InputInfo.BinaryCellInputInfo);
FrameObject fo = new FrameObject(OptimizerUtils.getUniqueTempFileName(), meta);
fo.acquireModify(frame);
fo.release();
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java b/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
index 6fe66ff..37bd309 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
@@ -39,7 +39,7 @@ import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
import org.apache.sysml.runtime.instructions.cp.Data;
import org.apache.sysml.runtime.instructions.cp.ScalarObject;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
import org.apache.sysml.utils.Explain.ExplainType;
import org.apache.sysml.utils.MLContextProxy;
@@ -537,8 +537,8 @@ public class MLContext {
exp.addVarParam(DataExpression.DATATYPEPARAM, new StringIdentifier("matrix", source));
exp.addVarParam(DataExpression.VALUETYPEPARAM, new StringIdentifier("double", source));
- if (mo.getMetaData() instanceof MatrixFormatMetaData) {
- MatrixFormatMetaData metaData = (MatrixFormatMetaData) mo.getMetaData();
+ if (mo.getMetaData() instanceof MetaDataFormat) {
+ MetaDataFormat metaData = (MetaDataFormat) mo.getMetaData();
if (metaData.getOutputInfo() == OutputInfo.CSVOutputInfo) {
exp.addVarParam(DataExpression.FORMAT_TYPE,
new StringIdentifier(DataExpression.FORMAT_TYPE_VALUE_CSV, source));
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java b/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java
index 6112a8a..41e8551 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java
@@ -56,7 +56,7 @@ import org.apache.sysml.runtime.instructions.spark.utils.FrameRDDConverterUtils;
import org.apache.sysml.runtime.instructions.spark.utils.RDDConverterUtils;
import org.apache.sysml.runtime.instructions.spark.utils.SparkUtils;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.IJV;
import org.apache.sysml.runtime.matrix.data.InputInfo;
@@ -111,7 +111,7 @@ public class MLContextConversionUtil {
ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
MatrixObject matrixObject = new MatrixObject(ValueType.DOUBLE, OptimizerUtils.getUniqueTempFileName(),
- new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo));
+ new MetaDataFormat(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo));
matrixObject.acquireModify(matrixBlock);
matrixObject.release();
@@ -165,7 +165,7 @@ public class MLContextConversionUtil {
MatrixCharacteristics mc = (matrixMetadata != null) ? matrixMetadata.asMatrixCharacteristics()
: new MatrixCharacteristics();
MatrixObject matrixObject = new MatrixObject(ValueType.DOUBLE, OptimizerUtils.getUniqueTempFileName(),
- new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo));
+ new MetaDataFormat(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo));
matrixObject.acquireModify(matrixBlock);
matrixObject.release();
return matrixObject;
@@ -190,7 +190,7 @@ public class MLContextConversionUtil {
try {
MatrixCharacteristics mc = (frameMetadata != null) ? frameMetadata.asMatrixCharacteristics()
: new MatrixCharacteristics();
- MatrixFormatMetaData mtd = new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo,
+ MetaDataFormat mtd = new MetaDataFormat(mc, OutputInfo.BinaryBlockOutputInfo,
InputInfo.BinaryBlockInputInfo);
FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(), mtd,
frameMetadata.getFrameSchema().getSchema().toArray(new ValueType[0]));
@@ -246,7 +246,7 @@ public class MLContextConversionUtil {
JavaPairRDD<MatrixIndexes, MatrixBlock> javaPairRdd = SparkUtils.copyBinaryBlockMatrix(binaryBlocks, copy);
MatrixObject matrixObject = new MatrixObject(ValueType.DOUBLE, OptimizerUtils.getUniqueTempFileName(),
- new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo));
+ new MetaDataFormat(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo));
matrixObject.setRDDHandle(new RDDObject(javaPairRdd, variableName));
return matrixObject;
}
@@ -315,7 +315,7 @@ public class MLContextConversionUtil {
UtilFunctions.nCopies((int)mc.getCols(), ValueType.STRING);
FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(),
- new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo), schema);
+ new MetaDataFormat(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo), schema);
frameObject.setRDDHandle(new RDDObject(binaryBlocks, variableName));
return frameObject;
}
@@ -618,7 +618,7 @@ public class MLContextConversionUtil {
: new MatrixCharacteristics();
MatrixObject matrixObject = new MatrixObject(ValueType.DOUBLE, OptimizerUtils.getUniqueTempFileName(),
- new MatrixFormatMetaData(mc, OutputInfo.CSVOutputInfo, InputInfo.CSVInputInfo));
+ new MetaDataFormat(mc, OutputInfo.CSVOutputInfo, InputInfo.CSVInputInfo));
JavaPairRDD<LongWritable, Text> javaPairRDD2 = javaPairRDD.mapToPair(new CopyTextInputFunction());
matrixObject.setRDDHandle(new RDDObject(javaPairRDD2, variableName));
return matrixObject;
@@ -656,7 +656,7 @@ public class MLContextConversionUtil {
JavaPairRDD<LongWritable, Text> javaPairRDDText = javaPairRDD.mapToPair(new CopyTextInputFunction());
FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(),
- new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo),
+ new MetaDataFormat(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo),
frameMetadata.getFrameSchema().getSchema().toArray(new ValueType[0]));
JavaPairRDD<Long, FrameBlock> rdd;
try {
@@ -689,7 +689,7 @@ public class MLContextConversionUtil {
: new MatrixCharacteristics();
MatrixObject matrixObject = new MatrixObject(ValueType.DOUBLE, OptimizerUtils.getUniqueTempFileName(),
- new MatrixFormatMetaData(mc, OutputInfo.TextCellOutputInfo, InputInfo.TextCellInputInfo));
+ new MetaDataFormat(mc, OutputInfo.TextCellOutputInfo, InputInfo.TextCellInputInfo));
JavaPairRDD<LongWritable, Text> javaPairRDD2 = javaPairRDD.mapToPair(new CopyTextInputFunction());
matrixObject.setRDDHandle(new RDDObject(javaPairRDD2, variableName));
return matrixObject;
@@ -716,7 +716,7 @@ public class MLContextConversionUtil {
JavaPairRDD<LongWritable, Text> javaPairRDDText = javaPairRDD.mapToPair(new CopyTextInputFunction());
FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(),
- new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo),
+ new MetaDataFormat(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo),
frameMetadata.getFrameSchema().getSchema().toArray(new ValueType[0]));
JavaPairRDD<Long, FrameBlock> rdd;
try {
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java b/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
index ea1e3d6..7b5f20e 100644
--- a/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
+++ b/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
@@ -60,7 +60,6 @@ import org.apache.sysml.runtime.instructions.cp.UnaryCPInstruction;
import org.apache.sysml.runtime.instructions.cp.VariableCPInstruction;
import org.apache.sysml.runtime.instructions.mr.MRInstruction;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixDimensionsMetaData;
import org.apache.sysml.runtime.matrix.operators.CMOperator;
import org.apache.sysml.runtime.matrix.operators.CMOperator.AggregateOperationTypes;
import org.apache.sysml.runtime.util.UtilFunctions;
@@ -227,7 +226,7 @@ public abstract class CostEstimator
if( dat instanceof MatrixObject ) //matrix
{
MatrixObject mo = (MatrixObject) dat;
- MatrixCharacteristics mc = ((MatrixDimensionsMetaData)mo.getMetaData()).getMatrixCharacteristics();
+ MatrixCharacteristics mc = mo.getMatrixCharacteristics();
long rlen = mc.getRows();
long clen = mc.getCols();
long brlen = mc.getRowsPerBlock();
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java b/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
index 2ab6b3c..de12bb1 100644
--- a/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
+++ b/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
@@ -58,7 +58,7 @@ import org.apache.sysml.runtime.instructions.cp.Data;
import org.apache.sysml.runtime.instructions.cp.ScalarObject;
import org.apache.sysml.runtime.instructions.cp.ScalarObjectFactory;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.udf.lib.DynamicReadMatrixCP;
import org.apache.sysml.udf.lib.DynamicReadMatrixRcCP;
import org.apache.sysml.udf.lib.OrderWrapper;
@@ -554,7 +554,7 @@ public class InterProceduralAnalysis
MatrixCharacteristics mc = new MatrixCharacteristics( input.getDim1(), input.getDim2(),
ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(),
fcallSizes.isSafeNnz(fkey, i)?input.getNnz():-1 );
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,null,null);
+ MetaDataFormat meta = new MetaDataFormat(mc,null,null);
mo.setMetaData(meta);
vars.put(dat.getName(), mo);
}
@@ -729,7 +729,7 @@ public class InterProceduralAnalysis
MatrixObject moOut = new MatrixObject(ValueType.DOUBLE, null);
MatrixCharacteristics mc = new MatrixCharacteristics( dim1, dim2,
ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), nnz);
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,null,null);
+ MetaDataFormat meta = new MetaDataFormat(mc,null,null);
moOut.setMetaData(meta);
return moOut;
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
index 3c1f6c9..c6d74dd 100644
--- a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
+++ b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
@@ -96,7 +96,7 @@ import org.apache.sysml.runtime.instructions.mr.RandInstruction;
import org.apache.sysml.runtime.instructions.mr.SeqInstruction;
import org.apache.sysml.runtime.io.IOUtilFunctions;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
@@ -1120,7 +1120,7 @@ public class Recompiler
int blksz = ConfigurationManager.getBlocksize();
MatrixCharacteristics mc = new MatrixCharacteristics(
dim1, dim2, blksz, blksz, nnz);
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,null,null);
+ MetaDataFormat meta = new MetaDataFormat(mc,null,null);
moOut.setMetaData(meta);
return moOut;
@@ -1357,7 +1357,7 @@ public class Recompiler
hop.getDim1(), hop.getDim2(),
ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(),
hop.getNnz());
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,null,null);
+ MetaDataFormat meta = new MetaDataFormat(mc,null,null);
mo.setMetaData(meta);
vars.put(varName, mo);
}
@@ -1751,7 +1751,7 @@ public class Recompiler
if( ret && !localMode ) {
for( MatrixObject mo : inputs )
{
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) mo.getMetaData();
+ MetaDataFormat iimd = (MetaDataFormat) mo.getMetaData();
if(( iimd.getInputInfo()==InputInfo.TextCellInputInfo
|| iimd.getInputInfo()==InputInfo.MatrixMarketInputInfo
|| iimd.getInputInfo()==InputInfo.CSVInputInfo
@@ -1805,7 +1805,7 @@ public class Recompiler
//robustness for usage through mlcontext (key/values of input rdds are
//not serializable for text; also bufferpool rdd read only supported for
// binarycell and binaryblock)
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) obj.getMetaData();
+ MetaDataFormat iimd = (MetaDataFormat) obj.getMetaData();
if( obj.getRDDHandle() != null
&& iimd.getInputInfo() != InputInfo.BinaryBlockInputInfo
&& iimd.getInputInfo() != InputInfo.BinaryCellInputInfo ) {
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemovePersistentReadWrite.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemovePersistentReadWrite.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemovePersistentReadWrite.java
index 34743bb..ec5c6c0 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemovePersistentReadWrite.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemovePersistentReadWrite.java
@@ -34,7 +34,7 @@ import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
import org.apache.sysml.runtime.controlprogram.caching.CacheableData;
import org.apache.sysml.runtime.instructions.cp.Data;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.MetaData;
import org.apache.sysml.runtime.matrix.data.InputInfo;
@@ -131,8 +131,8 @@ public class RewriteRemovePersistentReadWrite extends HopRewriteRule
//disable unnecessary reblock of binary block w/ equal block sizes
if( dop.requiresReblock() && _inputsMeta.containsKey(dop.getName())
- && _inputsMeta.get(dop.getName()) instanceof MatrixFormatMetaData) {
- MatrixFormatMetaData meta = (MatrixFormatMetaData)_inputsMeta.get(dop.getName());
+ && _inputsMeta.get(dop.getName()) instanceof MetaDataFormat) {
+ MetaDataFormat meta = (MetaDataFormat)_inputsMeta.get(dop.getName());
MatrixCharacteristics mc = meta.getMatrixCharacteristics();
boolean matchingBlksz = mc.getRowsPerBlock() == dop.getRowsInBlock()
&& mc.getColsPerBlock() == dop.getColsInBlock();
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/lops/runtime/RunMRJobs.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/runtime/RunMRJobs.java b/src/main/java/org/apache/sysml/lops/runtime/RunMRJobs.java
index 5cd0f60..2c04425 100644
--- a/src/main/java/org/apache/sysml/lops/runtime/RunMRJobs.java
+++ b/src/main/java/org/apache/sysml/lops/runtime/RunMRJobs.java
@@ -59,8 +59,7 @@ import org.apache.sysml.runtime.matrix.JobReturn;
import org.apache.sysml.runtime.matrix.MMCJMR;
import org.apache.sysml.runtime.matrix.MMRJMR;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixDimensionsMetaData;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.ReblockMR;
import org.apache.sysml.runtime.matrix.SortMR;
import org.apache.sysml.runtime.matrix.WriteCSVMR;
@@ -324,7 +323,7 @@ public class RunMRJobs
{
for (int i = 0; i < outputMatrices.length; i++) {
//get output meta data
- MatrixFormatMetaData meta = (MatrixFormatMetaData)outputMatrices[i].getMetaData();
+ MetaDataFormat meta = (MetaDataFormat)outputMatrices[i].getMetaData();
MatrixCharacteristics mc = meta.getMatrixCharacteristics();
OutputInfo outinfo = meta.getOutputInfo();
String fname = outputMatrices[i].getFileName();
@@ -345,7 +344,7 @@ public class RunMRJobs
// Currently, valueType information in not stored in MR instruction,
// since only DOUBLE matrices are supported ==> hard coded the value type information for now
MapReduceTool.writeMetaDataFile(fname + ".mtd", ValueType.DOUBLE,
- ((MatrixDimensionsMetaData)ret.getMetaData(i)).getMatrixCharacteristics(), outinfo);
+ ret.getMetaData(i).getMatrixCharacteristics(), outinfo);
}
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
index d84abfa..9738203 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
@@ -49,7 +49,7 @@ import org.apache.sysml.runtime.instructions.cp.ScalarObject;
import org.apache.sysml.runtime.instructions.cp.StringObject;
import org.apache.sysml.runtime.instructions.cp.VariableCPInstruction;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
import org.apache.sysml.udf.ExternalFunctionInvocationInstruction;
@@ -717,7 +717,7 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
throws CacheException
{
MatrixCharacteristics mc = new MatrixCharacteristics(m.getNumRows(),m.getNumCols(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
- MatrixFormatMetaData mfmd = new MatrixFormatMetaData(mc, OutputInfo.TextCellOutputInfo, InputInfo.TextCellInputInfo);
+ MetaDataFormat mfmd = new MetaDataFormat(mc, OutputInfo.TextCellOutputInfo, InputInfo.TextCellInputInfo);
return new MatrixObject(ValueType.DOUBLE, m.getFilePath(), mfmd);
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java
index 29806c2..8be1455 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java
@@ -31,7 +31,7 @@ import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
import org.apache.sysml.udf.ExternalFunctionInvocationInstruction;
@@ -168,7 +168,7 @@ public class ExternalFunctionProgramBlockCP extends ExternalFunctionProgramBlock
if( ret == null ) //otherwise, pass in-memory matrix from extfunct back to invoking program
{
MatrixCharacteristics mc = new MatrixCharacteristics(m.getNumRows(),m.getNumCols(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
- MatrixFormatMetaData mfmd = new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
+ MetaDataFormat mfmd = new MetaDataFormat(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
ret = new MatrixObject(ValueType.DOUBLE, m.getFilePath(), mfmd);
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
index c7ffdb1..e41b06c 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
@@ -47,13 +47,12 @@ import org.apache.sysml.runtime.instructions.spark.data.BroadcastObject;
import org.apache.sysml.runtime.instructions.spark.data.RDDObject;
import org.apache.sysml.runtime.io.IOUtilFunctions;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixDimensionsMetaData;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
+import org.apache.sysml.runtime.matrix.MetaDataNumItemsByEachReducer;
import org.apache.sysml.runtime.matrix.MetaData;
import org.apache.sysml.runtime.matrix.data.FileFormatProperties;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
-import org.apache.sysml.runtime.matrix.data.NumItemsByEachReducerMetaData;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
import org.apache.sysml.runtime.util.LocalFileUtils;
import org.apache.sysml.runtime.util.MapReduceTool;
@@ -314,8 +313,7 @@ public abstract class CacheableData<T extends CacheBlock> extends Data
}
public MatrixCharacteristics getMatrixCharacteristics() {
- MatrixDimensionsMetaData meta = (MatrixDimensionsMetaData) _metaData;
- return meta.getMatrixCharacteristics();
+ return _metaData.getMatrixCharacteristics();
}
public abstract void refreshMetaData()
@@ -990,7 +988,7 @@ public abstract class CacheableData<T extends CacheBlock> extends Data
protected T readBlobFromHDFS(String fname)
throws IOException
{
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) _metaData;
+ MetaDataFormat iimd = (MetaDataFormat) _metaData;
MatrixCharacteristics mc = iimd.getMatrixCharacteristics();
return readBlobFromHDFS(fname, mc.getRows(), mc.getCols());
}
@@ -1010,7 +1008,7 @@ public abstract class CacheableData<T extends CacheBlock> extends Data
protected void writeMetaData (String filePathAndName, String outputFormat, FileFormatProperties formatProperties)
throws DMLRuntimeException, IOException
{
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) _metaData;
+ MetaDataFormat iimd = (MetaDataFormat) _metaData;
if (iimd == null)
throw new DMLRuntimeException("Unexpected error while writing mtd file (" + filePathAndName + ") -- metadata is null.");
@@ -1042,7 +1040,7 @@ public abstract class CacheableData<T extends CacheBlock> extends Data
boolean ret = true;
if( outputFormat != null ) {
try {
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) _metaData;
+ MetaDataFormat iimd = (MetaDataFormat) _metaData;
OutputInfo oi1 = InputInfo.getMatchingOutputInfo( iimd.getInputInfo() );
OutputInfo oi2 = OutputInfo.stringToOutputInfo( outputFormat );
if( oi1 != oi2 )
@@ -1427,13 +1425,13 @@ public abstract class CacheableData<T extends CacheBlock> extends Data
str.append(": ");
str.append(_hdfsFileName + ", ");
- if (_metaData instanceof NumItemsByEachReducerMetaData) {
+ if (_metaData instanceof MetaDataNumItemsByEachReducer) {
str.append("NumItemsByEachReducerMetaData");
} else {
try {
- MatrixFormatMetaData md = (MatrixFormatMetaData) _metaData;
+ MetaDataFormat md = (MetaDataFormat) _metaData;
if (md != null) {
- MatrixCharacteristics mc = ((MatrixDimensionsMetaData) _metaData).getMatrixCharacteristics();
+ MatrixCharacteristics mc = _metaData.getMatrixCharacteristics();
str.append(mc.toString());
InputInfo ii = md.getInputInfo();
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
index 4cf3d7f..b594777 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
@@ -36,8 +36,7 @@ import org.apache.sysml.runtime.io.FrameReaderFactory;
import org.apache.sysml.runtime.io.FrameWriter;
import org.apache.sysml.runtime.io.FrameWriterFactory;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixDimensionsMetaData;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.MetaData;
import org.apache.sysml.runtime.matrix.data.FileFormatProperties;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
@@ -140,7 +139,7 @@ public class FrameObject extends CacheableData<FrameBlock>
throw new CacheException("Cannot refresh meta data because there is no data or meta data. ");
//update matrix characteristics
- MatrixCharacteristics mc = ((MatrixDimensionsMetaData) _metaData).getMatrixCharacteristics();
+ MatrixCharacteristics mc = _metaData.getMatrixCharacteristics();
mc.setDimension( _data.getNumRows(),_data.getNumColumns() );
mc.setNonZeros(_data.getNumRows()*_data.getNumColumns());
@@ -167,7 +166,7 @@ public class FrameObject extends CacheableData<FrameBlock>
protected FrameBlock readBlobFromHDFS(String fname, long rlen, long clen)
throws IOException
{
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) _metaData;
+ MetaDataFormat iimd = (MetaDataFormat) _metaData;
MatrixCharacteristics mc = iimd.getMatrixCharacteristics();
//handle missing schema if necessary
@@ -202,7 +201,7 @@ public class FrameObject extends CacheableData<FrameBlock>
//prepare return status (by default only collect)
status.setValue(false);
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) _metaData;
+ MetaDataFormat iimd = (MetaDataFormat) _metaData;
MatrixCharacteristics mc = iimd.getMatrixCharacteristics();
int rlen = (int)mc.getRows();
int clen = (int)mc.getCols();
@@ -247,7 +246,7 @@ public class FrameObject extends CacheableData<FrameBlock>
throws IOException, DMLRuntimeException
{
//prepare output info
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) _metaData;
+ MetaDataFormat iimd = (MetaDataFormat) _metaData;
OutputInfo oinfo = (ofmt != null ? OutputInfo.stringToOutputInfo (ofmt )
: InputInfo.getMatchingOutputInfo (iimd.getInputInfo ()));
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
index e9017e5..6486b02 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
@@ -35,8 +35,7 @@ import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartition
import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
import org.apache.sysml.runtime.instructions.spark.data.RDDObject;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixDimensionsMetaData;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.MetaData;
import org.apache.sysml.runtime.matrix.data.FileFormatProperties;
import org.apache.sysml.runtime.matrix.data.InputInfo;
@@ -115,8 +114,8 @@ public class MatrixObject extends CacheableData<MatrixBlock>
//base copy constructor
super(mo);
- MatrixFormatMetaData metaOld = (MatrixFormatMetaData)mo.getMetaData();
- _metaData = new MatrixFormatMetaData(new MatrixCharacteristics(metaOld.getMatrixCharacteristics()),
+ MetaDataFormat metaOld = (MetaDataFormat)mo.getMetaData();
+ _metaData = new MetaDataFormat(new MatrixCharacteristics(metaOld.getMatrixCharacteristics()),
metaOld.getOutputInfo(), metaOld.getInputInfo());
_updateType = mo._updateType;
@@ -136,7 +135,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
@Override
public void updateMatrixCharacteristics (MatrixCharacteristics mc) {
- ((MatrixDimensionsMetaData)_metaData).setMatrixCharacteristics( mc );
+ _metaData.getMatrixCharacteristics().set(mc);
}
/**
@@ -150,42 +149,35 @@ public class MatrixObject extends CacheableData<MatrixBlock>
{
if ( _data == null || _metaData ==null ) //refresh only for existing data
throw new CacheException("Cannot refresh meta data because there is no data or meta data. ");
- //we need to throw an exception, otherwise input/output format cannot be inferred
+ //we need to throw an exception, otherwise input/output format cannot be inferred
- MatrixCharacteristics mc = ((MatrixDimensionsMetaData) _metaData).getMatrixCharacteristics();
- mc.setDimension( _data.getNumRows(),
- _data.getNumColumns() );
- mc.setNonZeros( _data.getNonZeros() );
+ MatrixCharacteristics mc = _metaData.getMatrixCharacteristics();
+ mc.setDimension( _data.getNumRows(), _data.getNumColumns() );
+ mc.setNonZeros( _data.getNonZeros() );
}
public long getNumRows() {
- MatrixCharacteristics mc = getMatrixCharacteristics();
- return mc.getRows ();
+ return getMatrixCharacteristics().getRows();
}
public long getNumColumns() {
- MatrixCharacteristics mc = getMatrixCharacteristics();
- return mc.getCols ();
+ return getMatrixCharacteristics().getCols();
}
public long getNumRowsPerBlock() {
- MatrixCharacteristics mc = getMatrixCharacteristics();
- return mc.getRowsPerBlock();
+ return getMatrixCharacteristics().getRowsPerBlock();
}
public long getNumColumnsPerBlock() {
- MatrixCharacteristics mc = getMatrixCharacteristics();
- return mc.getColsPerBlock();
+ return getMatrixCharacteristics().getColsPerBlock();
}
public long getNnz() {
- MatrixCharacteristics mc = getMatrixCharacteristics();
- return mc.getNonZeros();
+ return getMatrixCharacteristics().getNonZeros();
}
public double getSparsity() {
- MatrixCharacteristics mc = getMatrixCharacteristics();
- return OptimizerUtils.getSparsity(mc);
+ return OptimizerUtils.getSparsity(getMatrixCharacteristics());
}
// *********************************************
@@ -264,7 +256,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
boolean blockwise = (_partitionFormat==PDataPartitionFormat.ROW_BLOCK_WISE || _partitionFormat==PDataPartitionFormat.COLUMN_BLOCK_WISE);
//preparations for block wise access
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) _metaData;
+ MetaDataFormat iimd = (MetaDataFormat) _metaData;
MatrixCharacteristics mc = iimd.getMatrixCharacteristics();
int brlen = mc.getRowsPerBlock();
int bclen = mc.getColsPerBlock();
@@ -427,7 +419,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
protected MatrixBlock readBlobFromHDFS(String fname, long rlen, long clen)
throws IOException
{
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) _metaData;
+ MetaDataFormat iimd = (MetaDataFormat) _metaData;
MatrixCharacteristics mc = iimd.getMatrixCharacteristics();
long begin = 0;
@@ -464,7 +456,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
//prepare return status (by default only collect)
writeStatus.setValue(false);
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) _metaData;
+ MetaDataFormat iimd = (MetaDataFormat) _metaData;
MatrixCharacteristics mc = iimd.getMatrixCharacteristics();
InputInfo ii = iimd.getInputInfo();
MatrixBlock mb = null;
@@ -489,7 +481,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
//note: lazy, partition-at-a-time collect (toLocalIterator) was significantly slower
if( !MapReduceTool.existsFileOnHDFS(_hdfsFileName) ) { //prevent overwrite existing file
long newnnz = SparkExecutionContext.writeRDDtoHDFS(lrdd, _hdfsFileName, iimd.getOutputInfo());
- ((MatrixDimensionsMetaData) _metaData).getMatrixCharacteristics().setNonZeros(newnnz);
+ _metaData.getMatrixCharacteristics().setNonZeros(newnnz);
((RDDObject)rdd).setPending(false); //mark rdd as non-pending (for export)
((RDDObject)rdd).setHDFSFile(true); //mark rdd as hdfs file (for restore)
writeStatus.setValue(true); //mark for no cache-write on read
@@ -533,7 +525,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
begin = System.currentTimeMillis();
}
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) _metaData;
+ MetaDataFormat iimd = (MetaDataFormat) _metaData;
if (_data != null)
{
@@ -569,14 +561,14 @@ public class MatrixObject extends CacheableData<MatrixBlock>
protected void writeBlobFromRDDtoHDFS(RDDObject rdd, String fname, String outputFormat)
throws IOException, DMLRuntimeException
{
- //prepare output info
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) _metaData;
- OutputInfo oinfo = (outputFormat != null ? OutputInfo.stringToOutputInfo (outputFormat)
- : InputInfo.getMatchingOutputInfo (iimd.getInputInfo ()));
-
+ //prepare output info
+ MetaDataFormat iimd = (MetaDataFormat) _metaData;
+ OutputInfo oinfo = (outputFormat != null ? OutputInfo.stringToOutputInfo (outputFormat) :
+ InputInfo.getMatchingOutputInfo (iimd.getInputInfo ()));
+
//note: the write of an RDD to HDFS might trigger
//lazy evaluation of pending transformations.
long newnnz = SparkExecutionContext.writeRDDtoHDFS(rdd, fname, oinfo);
- ((MatrixDimensionsMetaData) _metaData).getMatrixCharacteristics().setNonZeros(newnnz);
+ _metaData.getMatrixCharacteristics().setNonZeros(newnnz);
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
index 4cd371b..79a658d 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
@@ -48,8 +48,7 @@ import org.apache.sysml.runtime.instructions.cp.ScalarObjectFactory;
import org.apache.sysml.runtime.instructions.gpu.context.GPUContext;
import org.apache.sysml.runtime.instructions.gpu.context.GPUObject;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixDimensionsMetaData;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.MetaData;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
@@ -233,8 +232,7 @@ public class ExecutionContext {
public MatrixCharacteristics getMatrixCharacteristics( String varname )
throws DMLRuntimeException
{
- MatrixDimensionsMetaData dims = (MatrixDimensionsMetaData) getMetaData(varname);
- return dims.getMatrixCharacteristics();
+ return getMetaData(varname).getMatrixCharacteristics();
}
/**
@@ -280,14 +278,14 @@ public class ExecutionContext {
return;
MetaData oldMetaData = mo.getMetaData();
- if( oldMetaData == null || !(oldMetaData instanceof MatrixFormatMetaData) )
+ if( oldMetaData == null || !(oldMetaData instanceof MetaDataFormat) )
throw new DMLRuntimeException("Metadata not available");
MatrixCharacteristics mc = new MatrixCharacteristics((long)nrows, (long)ncols,
(int) mo.getNumRowsPerBlock(), (int)mo.getNumColumnsPerBlock());
- mo.setMetaData(new MatrixFormatMetaData(mc,
- ((MatrixFormatMetaData)oldMetaData).getOutputInfo(),
- ((MatrixFormatMetaData)oldMetaData).getInputInfo()));
+ mo.setMetaData(new MetaDataFormat(mc,
+ ((MetaDataFormat)oldMetaData).getOutputInfo(),
+ ((MetaDataFormat)oldMetaData).getInputInfo()));
}
/**
@@ -611,28 +609,22 @@ public class ExecutionContext {
public void cleanupMatrixObject(MatrixObject mo)
throws DMLRuntimeException
{
- try
- {
- if ( mo.isCleanupEnabled() )
- {
- //compute ref count only if matrix cleanup actually necessary
- if ( !getVariables().hasReferences(mo) ) {
- //clean cached data
- mo.clearData();
- if( mo.isHDFSFileExists() )
- {
- //clean hdfs data
- String fpath = mo.getFileName();
- if (fpath != null) {
- MapReduceTool.deleteFileIfExistOnHDFS(fpath);
- MapReduceTool.deleteFileIfExistOnHDFS(fpath + ".mtd");
- }
- }
+ //early abort w/o scan of symbol table if no cleanup required
+ boolean fileExists = (mo.isHDFSFileExists() && mo.getFileName() != null);
+ if( !CacheableData.isCachingActive() && !fileExists )
+ return;
+
+ try {
+ //compute ref count only if matrix cleanup actually necessary
+ if ( mo.isCleanupEnabled() && !getVariables().hasReferences(mo) ) {
+ mo.clearData(); //clean cached data
+ if( fileExists ) {
+ MapReduceTool.deleteFileIfExistOnHDFS(mo.getFileName());
+ MapReduceTool.deleteFileIfExistOnHDFS(mo.getFileName()+".mtd");
}
}
}
- catch(Exception ex)
- {
+ catch(Exception ex) {
throw new DMLRuntimeException(ex);
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitioner.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitioner.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitioner.java
index d836e18..69d2a70 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitioner.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitioner.java
@@ -29,7 +29,7 @@ import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
@@ -102,7 +102,7 @@ public abstract class DataPartitioner
return in;
//analyze input matrix object
- MatrixFormatMetaData meta = (MatrixFormatMetaData)in.getMetaData();
+ MetaDataFormat meta = (MetaDataFormat)in.getMetaData();
MatrixCharacteristics mc = meta.getMatrixCharacteristics();
InputInfo ii = meta.getInputInfo();
OutputInfo oi = meta.getOutputInfo();
@@ -167,7 +167,7 @@ public abstract class DataPartitioner
mcNew.setNonZeros( nonZeros );
if( convertBlock2Cell )
ii = InputInfo.BinaryCellInputInfo;
- MatrixFormatMetaData metaNew = new MatrixFormatMetaData(mcNew,oi,ii);
+ MetaDataFormat metaNew = new MetaDataFormat(mcNew,oi,ii);
out.setMetaData(metaNew);
return out;
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
index 0effc7f..860c7b6 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
@@ -84,7 +84,7 @@ import org.apache.sysml.runtime.instructions.gpu.GPUInstruction;
import org.apache.sysml.runtime.instructions.mr.MRInstruction;
import org.apache.sysml.runtime.instructions.spark.SPInstruction;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
@@ -878,7 +878,7 @@ public class ProgramConverter
break;
case MATRIX:
MatrixObject mo = (MatrixObject) dat;
- MatrixFormatMetaData md = (MatrixFormatMetaData) dat.getMetaData();
+ MetaDataFormat md = (MetaDataFormat) dat.getMetaData();
MatrixCharacteristics mc = md.getMatrixCharacteristics();
value = mo.getFileName();
PartitionFormat partFormat = (mo.getPartitionFormat()!=null) ? new PartitionFormat(
@@ -1750,7 +1750,7 @@ public class ProgramConverter
PartitionFormat partFormat = PartitionFormat.valueOf( st.nextToken() );
UpdateType inplace = UpdateType.valueOf( st.nextToken() );
MatrixCharacteristics mc = new MatrixCharacteristics(rows, cols, brows, bcols, nnz);
- MatrixFormatMetaData md = new MatrixFormatMetaData( mc, oin, iin );
+ MetaDataFormat md = new MetaDataFormat( mc, oin, iin );
mo.setMetaData( md );
mo.setVarName( name );
if( partFormat._dpf != PDataPartitionFormat.NONE )
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalFile.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalFile.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalFile.java
index cd0b3b7..253883b 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalFile.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalFile.java
@@ -51,7 +51,7 @@ import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
import org.apache.sysml.runtime.controlprogram.parfor.util.StagingFileUtils;
import org.apache.sysml.runtime.io.IOUtilFunctions;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.IJV;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
@@ -157,7 +157,7 @@ public class ResultMergeLocalFile extends ResultMerge
{
String varName = _output.getVarName();
ValueType vt = _output.getValueType();
- MatrixFormatMetaData metadata = (MatrixFormatMetaData) _output.getMetaData();
+ MetaDataFormat metadata = (MetaDataFormat) _output.getMetaData();
MatrixObject moNew = new MatrixObject( vt, _outputFName );
moNew.setVarName( varName.contains(NAME_SUFFIX) ? varName : varName+NAME_SUFFIX );
@@ -170,7 +170,7 @@ public class ResultMergeLocalFile extends ResultMerge
MatrixCharacteristics mc = new MatrixCharacteristics(mcOld.getRows(),mcOld.getCols(),
mcOld.getRowsPerBlock(),mcOld.getColsPerBlock());
mc.setNonZeros( computeNonZeros(output, inMO) );
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,oiOld,iiOld);
+ MetaDataFormat meta = new MetaDataFormat(mc,oiOld,iiOld);
moNew.setMetaData( meta );
return moNew;
@@ -179,7 +179,7 @@ public class ResultMergeLocalFile extends ResultMerge
private void merge( String fnameNew, MatrixObject outMo, ArrayList<MatrixObject> inMO )
throws DMLRuntimeException
{
- OutputInfo oi = ((MatrixFormatMetaData)outMo.getMetaData()).getOutputInfo();
+ OutputInfo oi = ((MetaDataFormat)outMo.getMetaData()).getOutputInfo();
boolean withCompare = ( outMo.getNnz() != 0 ); //if nnz exist or unknown (-1)
if( oi == OutputInfo.TextCellOutputInfo )
@@ -295,7 +295,7 @@ public class ResultMergeLocalFile extends ResultMerge
}
//Step 2) read blocks, consolidate, and write to HDFS
- createTextCellResultFile(fnameStaging, fnameStagingCompare, fnameNew, (MatrixFormatMetaData)outMo.getMetaData(), true);
+ createTextCellResultFile(fnameStaging, fnameStagingCompare, fnameNew, (MetaDataFormat)outMo.getMetaData(), true);
}
catch(Exception ex)
{
@@ -390,7 +390,7 @@ public class ResultMergeLocalFile extends ResultMerge
}
//Step 2) read blocks, consolidate, and write to HDFS
- createBinaryCellResultFile(fnameStaging, fnameStagingCompare, fnameNew, (MatrixFormatMetaData)outMo.getMetaData(), true);
+ createBinaryCellResultFile(fnameStaging, fnameStagingCompare, fnameNew, (MetaDataFormat)outMo.getMetaData(), true);
}
catch(Exception ex)
{
@@ -420,7 +420,7 @@ public class ResultMergeLocalFile extends ResultMerge
}
//Step 2) read blocks, consolidate, and write to HDFS
- createBinaryBlockResultFile(fnameStaging, null, fnameNew, (MatrixFormatMetaData)outMo.getMetaData(), false);
+ createBinaryBlockResultFile(fnameStaging, null, fnameNew, (MetaDataFormat)outMo.getMetaData(), false);
}
catch(Exception ex)
{
@@ -454,7 +454,7 @@ public class ResultMergeLocalFile extends ResultMerge
}
//Step 2) read blocks, consolidate, and write to HDFS
- createBinaryBlockResultFile(fnameStaging, fnameStagingCompare, fnameNew, (MatrixFormatMetaData)outMo.getMetaData(), true);
+ createBinaryBlockResultFile(fnameStaging, fnameStagingCompare, fnameNew, (MetaDataFormat)outMo.getMetaData(), true);
}
catch(Exception ex)
{
@@ -644,7 +644,7 @@ public class ResultMergeLocalFile extends ResultMerge
}
@SuppressWarnings("deprecation")
- private void createBinaryBlockResultFile( String fnameStaging, String fnameStagingCompare, String fnameNew, MatrixFormatMetaData metadata, boolean withCompare )
+ private void createBinaryBlockResultFile( String fnameStaging, String fnameStagingCompare, String fnameNew, MetaDataFormat metadata, boolean withCompare )
throws IOException, DMLRuntimeException
{
JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
@@ -740,7 +740,7 @@ public class ResultMergeLocalFile extends ResultMerge
}
}
- private void createTextCellResultFile( String fnameStaging, String fnameStagingCompare, String fnameNew, MatrixFormatMetaData metadata, boolean withCompare )
+ private void createTextCellResultFile( String fnameStaging, String fnameStagingCompare, String fnameNew, MetaDataFormat metadata, boolean withCompare )
throws IOException, DMLRuntimeException
{
JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
@@ -877,7 +877,7 @@ public class ResultMergeLocalFile extends ResultMerge
}
@SuppressWarnings("deprecation")
- private void createBinaryCellResultFile( String fnameStaging, String fnameStagingCompare, String fnameNew, MatrixFormatMetaData metadata, boolean withCompare )
+ private void createBinaryCellResultFile( String fnameStaging, String fnameStagingCompare, String fnameNew, MetaDataFormat metadata, boolean withCompare )
throws IOException, DMLRuntimeException
{
JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalMemory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalMemory.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalMemory.java
index 638a845..5fcfb26 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalMemory.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalMemory.java
@@ -27,7 +27,7 @@ import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
@@ -228,7 +228,7 @@ public class ResultMergeLocalMemory extends ResultMerge
{
String varName = _output.getVarName();
ValueType vt = _output.getValueType();
- MatrixFormatMetaData metadata = (MatrixFormatMetaData) _output.getMetaData();
+ MetaDataFormat metadata = (MetaDataFormat) _output.getMetaData();
MatrixObject moNew = new MatrixObject( vt, _outputFName );
moNew.setVarName( varName.contains(NAME_SUFFIX) ? varName : varName+NAME_SUFFIX );
@@ -241,7 +241,7 @@ public class ResultMergeLocalMemory extends ResultMerge
MatrixCharacteristics mc = new MatrixCharacteristics(mcOld.getRows(),mcOld.getCols(),
mcOld.getRowsPerBlock(),mcOld.getColsPerBlock());
mc.setNonZeros(data.getNonZeros());
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,oiOld,iiOld);
+ MetaDataFormat meta = new MetaDataFormat(mc,oiOld,iiOld);
moNew.setMetaData( meta );
//adjust dense/sparse representation
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java
index 909d0d0..76cbd40 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java
@@ -41,7 +41,7 @@ import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
import org.apache.sysml.runtime.controlprogram.parfor.util.StagingFileUtils;
import org.apache.sysml.runtime.io.IOUtilFunctions;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixCell;
@@ -123,7 +123,7 @@ public class ResultMergeRemoteMR extends ResultMerge
_output.exportData();
//actual merge
- MatrixFormatMetaData metadata = (MatrixFormatMetaData) _output.getMetaData();
+ MetaDataFormat metadata = (MetaDataFormat) _output.getMetaData();
MatrixCharacteristics mcOld = metadata.getMatrixCharacteristics();
String fnameCompare = _output.getFileName();
@@ -145,7 +145,7 @@ public class ResultMergeRemoteMR extends ResultMerge
MatrixCharacteristics mc = new MatrixCharacteristics(mcOld.getRows(),mcOld.getCols(),
mcOld.getRowsPerBlock(),mcOld.getColsPerBlock());
mc.setNonZeros( computeNonZeros(_output, inMO) );
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,oiOld,iiOld);
+ MetaDataFormat meta = new MetaDataFormat(mc,oiOld,iiOld);
moNew.setMetaData( meta );
}
else
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSpark.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSpark.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSpark.java
index 95dbb8b..9accfea 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSpark.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSpark.java
@@ -38,7 +38,7 @@ import org.apache.sysml.runtime.instructions.spark.data.RDDObject;
import org.apache.sysml.runtime.instructions.spark.functions.CopyBlockPairFunction;
import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
@@ -86,7 +86,7 @@ public class ResultMergeRemoteSpark extends ResultMerge
if( _inputs != null && _inputs.length>0 )
{
//prepare compare
- MatrixFormatMetaData metadata = (MatrixFormatMetaData) _output.getMetaData();
+ MetaDataFormat metadata = (MetaDataFormat) _output.getMetaData();
MatrixCharacteristics mcOld = metadata.getMatrixCharacteristics();
MatrixObject compare = (mcOld.getNonZeros()==0) ? null : _output;
@@ -104,7 +104,7 @@ public class ResultMergeRemoteSpark extends ResultMerge
MatrixCharacteristics mc = new MatrixCharacteristics(mcOld.getRows(),mcOld.getCols(),
mcOld.getRowsPerBlock(),mcOld.getColsPerBlock());
mc.setNonZeros( computeNonZeros(_output, Arrays.asList(_inputs)) );
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,oiOld,iiOld);
+ MetaDataFormat meta = new MetaDataFormat(mc,oiOld,iiOld);
moNew.setMetaData( meta );
moNew.setRDDHandle( ro );
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
index 0d0c52e..1294809 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
@@ -94,7 +94,7 @@ import org.apache.sysml.runtime.instructions.cp.FunctionCallCPInstruction;
import org.apache.sysml.runtime.instructions.gpu.context.GPUContextPool;
import org.apache.sysml.runtime.instructions.spark.data.RDDObject;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
import org.apache.sysml.runtime.matrix.data.SparseRowVector;
@@ -2247,7 +2247,7 @@ public class OptimizerRuleBased extends Optimizer
else
{
MatrixObject mo = (MatrixObject)dat;
- MatrixFormatMetaData meta = (MatrixFormatMetaData) mo.getMetaData();
+ MetaDataFormat meta = (MetaDataFormat) mo.getMetaData();
OutputInfo oi = meta.getOutputInfo();
long nnz = meta.getMatrixCharacteristics().getNonZeros();
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/instructions/MRJobInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/MRJobInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/MRJobInstruction.java
index 3dcad83..eceebba 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/MRJobInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/MRJobInstruction.java
@@ -40,10 +40,9 @@ import org.apache.sysml.runtime.controlprogram.parfor.ProgramConverter;
import org.apache.sysml.runtime.instructions.cp.Data;
import org.apache.sysml.runtime.matrix.JobReturn;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixDimensionsMetaData;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
+import org.apache.sysml.runtime.matrix.MetaDataNumItemsByEachReducer;
import org.apache.sysml.runtime.matrix.data.InputInfo;
-import org.apache.sysml.runtime.matrix.data.NumItemsByEachReducerMetaData;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
import org.apache.sysml.runtime.util.UtilFunctions;
@@ -895,10 +894,10 @@ public class MRJobInstruction extends Instruction
clens[i] = mc.getCols();
brlens[i] = mc.getRowsPerBlock();
bclens[i] = mc.getColsPerBlock();
- if ( inputMatrices[i].getMetaData() instanceof MatrixFormatMetaData ) {
- inputInfos[i] = ((MatrixFormatMetaData) inputMatrices[i].getMetaData()).getInputInfo();
+ if ( inputMatrices[i].getMetaData() instanceof MetaDataFormat ) {
+ inputInfos[i] = ((MetaDataFormat) inputMatrices[i].getMetaData()).getInputInfo();
}
- else if (inputMatrices[i].getMetaData() instanceof NumItemsByEachReducerMetaData ) {
+ else if (inputMatrices[i].getMetaData() instanceof MetaDataNumItemsByEachReducer ) {
inputInfos[i] = InputInfo.InputInfoForSortOutput;
inputInfos[i].metadata = inputMatrices[i].getMetaData();
}
@@ -923,7 +922,7 @@ public class MRJobInstruction extends Instruction
// Populate information
for(int i=0; i < outputVars.length; i++) {
outputs[i] = outputMatrices[i].getFileName();
- MatrixFormatMetaData md = (MatrixFormatMetaData) outputMatrices[i].getMetaData();
+ MetaDataFormat md = (MetaDataFormat) outputMatrices[i].getMetaData();
outputInfos[i] = md.getOutputInfo();
}
}
@@ -1294,7 +1293,7 @@ public class MRJobInstruction extends Instruction
/* Populate returned stats into symbol table of matrices */
for ( int index=0; index < jb.getMetaData().length; index++) {
String varname = getOutputVars()[index];
- MatrixCharacteristics mc = ((MatrixDimensionsMetaData)jb.getMetaData(index)).getMatrixCharacteristics();
+ MatrixCharacteristics mc = jb.getMetaData(index).getMatrixCharacteristics();
ec.getVariable(varname).updateMatrixCharacteristics(mc);
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/instructions/cp/DataPartitionCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/DataPartitionCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/DataPartitionCPInstruction.java
index f52b8f4..6bb6fcd 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/DataPartitionCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/DataPartitionCPInstruction.java
@@ -30,7 +30,7 @@ import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysml.runtime.instructions.InstructionUtils;
import org.apache.sysml.runtime.io.WriterBinaryBlock;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
@@ -85,7 +85,7 @@ public class DataPartitionCPInstruction extends UnaryCPInstruction {
//ensure correctness of output characteristics (required if input unknown during compile and no recompile)
MatrixCharacteristics mc = new MatrixCharacteristics(moIn.getNumRows(), moIn.getNumColumns(), (int)moIn.getNumRowsPerBlock(), (int)moIn.getNumColumnsPerBlock(), moIn.getNnz());
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
+ MetaDataFormat meta = new MetaDataFormat(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
moOut.setMetaData(meta);
}
catch(Exception ex)
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/instructions/cp/QuantilePickCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/QuantilePickCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/QuantilePickCPInstruction.java
index cace173..1ad6054 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/QuantilePickCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/QuantilePickCPInstruction.java
@@ -28,8 +28,8 @@ import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysml.runtime.instructions.InstructionUtils;
import org.apache.sysml.runtime.matrix.MetaData;
+import org.apache.sysml.runtime.matrix.MetaDataNumItemsByEachReducer;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
-import org.apache.sysml.runtime.matrix.data.NumItemsByEachReducerMetaData;
import org.apache.sysml.runtime.matrix.operators.Operator;
import org.apache.sysml.runtime.util.MapReduceTool;
import org.apache.sysml.runtime.util.UtilFunctions;
@@ -130,7 +130,7 @@ public class QuantilePickCPInstruction extends BinaryCPInstruction {
if ( mdata != null ) {
try {
- double picked = MapReduceTool.pickValue(fname, (NumItemsByEachReducerMetaData) mdata, pickindex.getDoubleValue());
+ double picked = MapReduceTool.pickValue(fname, (MetaDataNumItemsByEachReducer) mdata, pickindex.getDoubleValue());
ec.setVariable(output.getName(), new DoubleObject(picked));
} catch (Exception e ) {
throw new DMLRuntimeException(e);
@@ -158,7 +158,7 @@ public class QuantilePickCPInstruction extends BinaryCPInstruction {
if ( mdata1 != null ) {
try {
- double median = MapReduceTool.median(fname1, (NumItemsByEachReducerMetaData) mdata1);
+ double median = MapReduceTool.median(fname1, (MetaDataNumItemsByEachReducer) mdata1);
ec.setVariable(output.getName(), new DoubleObject(median));
} catch (Exception e ) {
throw new DMLRuntimeException(e);
@@ -186,13 +186,13 @@ public class QuantilePickCPInstruction extends BinaryCPInstruction {
double[] q25 = null;
double[] q75 = null;
try {
- q25 = MapReduceTool.pickValueWeight(inputMatrix.getFileName(), (NumItemsByEachReducerMetaData) inputMatrix.getMetaData(), 0.25, false);
- q75 = MapReduceTool.pickValueWeight(inputMatrix.getFileName(), (NumItemsByEachReducerMetaData) inputMatrix.getMetaData(), 0.75, false);
+ q25 = MapReduceTool.pickValueWeight(inputMatrix.getFileName(), (MetaDataNumItemsByEachReducer) inputMatrix.getMetaData(), 0.25, false);
+ q75 = MapReduceTool.pickValueWeight(inputMatrix.getFileName(), (MetaDataNumItemsByEachReducer) inputMatrix.getMetaData(), 0.75, false);
} catch (IOException e1) {
throw new DMLRuntimeException(e1);
}
- double sumwt = UtilFunctions.getTotalLength((NumItemsByEachReducerMetaData) ec.getMetaData(input1.getName()));
+ double sumwt = UtilFunctions.getTotalLength((MetaDataNumItemsByEachReducer) ec.getMetaData(input1.getName()));
double q25d = sumwt*0.25;
double q75d = sumwt*0.75;
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
index 1c0f9e6..2239919 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
@@ -46,7 +46,7 @@ import org.apache.sysml.runtime.io.IOUtilFunctions;
import org.apache.sysml.runtime.io.WriterMatrixMarket;
import org.apache.sysml.runtime.io.WriterTextCSV;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.MetaData;
import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
import org.apache.sysml.runtime.matrix.data.FileFormatProperties;
@@ -360,7 +360,7 @@ public class VariableCPInstruction extends CPInstruction {
else {
throw new DMLRuntimeException("Invalid number of operands in createvar instruction: " + str);
}
- MatrixFormatMetaData iimd = new MatrixFormatMetaData(mc, oi, ii);
+ MetaDataFormat iimd = new MetaDataFormat(mc, oi, ii);
UpdateType updateType = UpdateType.COPY;
if ( parts.length >= 12 )
updateType = UpdateType.valueOf(parts[11].toUpperCase());
@@ -489,16 +489,12 @@ public class VariableCPInstruction extends CPInstruction {
//create new variable for symbol table and cache
//(existing objects gets cleared through rmvar instructions)
String fname = getInput2().getName();
-
// check if unique filename needs to be generated
- boolean overrideFileName = ((BooleanObject) ec.getScalarInput(getInput3().getName(), getInput3().getValueType(), true)).getBooleanValue();
- if ( overrideFileName ) {
+ if( Boolean.parseBoolean(getInput3().getName()) )
fname = fname + "_" + _uniqueVarID.getNextID();
- }
MatrixObject mobj = new MatrixObject(getInput1().getValueType(), fname );
mobj.setVarName(getInput1().getName());
- mobj.setDataType(DataType.MATRIX);
//clone meta data because it is updated on copy-on-write, otherwise there
//is potential for hidden side effects between variables.
mobj.setMetaData((MetaData)metadata.clone());
@@ -512,7 +508,6 @@ public class VariableCPInstruction extends CPInstruction {
String fname = getInput2().getName();
FrameObject fobj = new FrameObject(fname);
fobj.setVarName(getInput1().getName());
- fobj.setDataType(DataType.FRAME);
fobj.setMetaData((MetaData)metadata.clone());
fobj.setFileFormatProperties(_formatProperties);
if( _schema != null )
@@ -520,8 +515,8 @@ public class VariableCPInstruction extends CPInstruction {
ec.setVariable(getInput1().getName(), fobj);
}
else if ( getInput1().getDataType() == DataType.SCALAR ){
- ScalarObject sobj = null;
- ec.setScalarOutput(getInput1().getName(), sobj);
+ //created variable not called for scalars
+ ec.setScalarOutput(getInput1().getName(), null);
}
else {
throw new DMLRuntimeException("Unexpected data type: " + getInput1().getDataType());
@@ -862,8 +857,8 @@ public class VariableCPInstruction extends CPInstruction {
}
else {
try {
- OutputInfo oi = ((MatrixFormatMetaData)mo.getMetaData()).getOutputInfo();
- MatrixCharacteristics mc = ((MatrixFormatMetaData)mo.getMetaData()).getMatrixCharacteristics();
+ OutputInfo oi = ((MetaDataFormat)mo.getMetaData()).getOutputInfo();
+ MatrixCharacteristics mc = ((MetaDataFormat)mo.getMetaData()).getMatrixCharacteristics();
if(oi == OutputInfo.CSVOutputInfo) {
WriterTextCSV writer = new WriterTextCSV((CSVFileFormatProperties)_formatProperties);
writer.addHeaderToCSV(mo.getFileName(), fname, mc.getRows(), mc.getCols());
@@ -901,7 +896,7 @@ public class VariableCPInstruction extends CPInstruction {
mo.exportData(fname, outFmt);
}
else {
- OutputInfo oi = ((MatrixFormatMetaData)mo.getMetaData()).getOutputInfo();
+ OutputInfo oi = ((MetaDataFormat)mo.getMetaData()).getOutputInfo();
MatrixCharacteristics mc = mo.getMatrixCharacteristics();
if(oi == OutputInfo.TextCellOutputInfo) {
try {
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/instructions/cpfile/MatrixIndexingCPFileInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cpfile/MatrixIndexingCPFileInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cpfile/MatrixIndexingCPFileInstruction.java
index acfd4d4..0b35837 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cpfile/MatrixIndexingCPFileInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cpfile/MatrixIndexingCPFileInstruction.java
@@ -29,7 +29,7 @@ import org.apache.sysml.runtime.instructions.InstructionUtils;
import org.apache.sysml.runtime.instructions.cp.CPOperand;
import org.apache.sysml.runtime.instructions.cp.IndexingCPInstruction;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.operators.Operator;
import org.apache.sysml.runtime.matrix.operators.SimpleOperator;
@@ -92,7 +92,7 @@ public final class MatrixIndexingCPFileInstruction extends IndexingCPInstruction
if( mo.isPartitioned() && opcode.equalsIgnoreCase(RightIndex.OPCODE) )
{
- MatrixFormatMetaData meta = (MatrixFormatMetaData)mo.getMetaData();
+ MetaDataFormat meta = (MetaDataFormat)mo.getMetaData();
MatrixCharacteristics mc = meta.getMatrixCharacteristics();
String pfname = mo.getPartitionFileName( ixrange, mc.getRowsPerBlock(), mc.getColsPerBlock());
@@ -123,7 +123,7 @@ public final class MatrixIndexingCPFileInstruction extends IndexingCPInstruction
throw new DMLRuntimeException("Unsupported partition format for CP_FILE "+RightIndex.OPCODE+": "+ mo.getPartitionFormat());
}
- MatrixFormatMetaData metaNew = new MatrixFormatMetaData(mcNew,meta.getOutputInfo(),meta.getInputInfo());
+ MetaDataFormat metaNew = new MetaDataFormat(mcNew,meta.getOutputInfo(),meta.getInputInfo());
mobj.setMetaData(metaNew);
//put output object into symbol table
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java
index a85b7ad..fed3ad5 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java
@@ -59,7 +59,7 @@ import org.apache.sysml.runtime.instructions.cp.ParameterizedBuiltinCPInstructio
import org.apache.sysml.runtime.io.IOUtilFunctions;
import org.apache.sysml.runtime.io.MatrixWriter;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixCell;
@@ -161,7 +161,7 @@ public class ParameterizedBuiltinCPFileInstruction extends ParameterizedBuiltinC
//initial setup
String fnameOld = _src.getFileName();
String fnameNew = _out.getFileName();
- InputInfo ii = ((MatrixFormatMetaData)_src.getMetaData()).getInputInfo();
+ InputInfo ii = ((MetaDataFormat)_src.getMetaData()).getInputInfo();
MatrixCharacteristics mc = _src.getMatrixCharacteristics();
String stagingDir = LocalFileUtils.getUniqueWorkingDir(LocalFileUtils.CATEGORY_WORK);
@@ -228,7 +228,7 @@ public class ParameterizedBuiltinCPFileInstruction extends ParameterizedBuiltinC
String varName = out.getVarName();
String fName = out.getFileName();
ValueType vt = src.getValueType();
- MatrixFormatMetaData metadata = (MatrixFormatMetaData) src.getMetaData();
+ MetaDataFormat metadata = (MetaDataFormat) src.getMetaData();
MatrixObject moNew = new MatrixObject( vt, fName );
moNew.setVarName( varName );
@@ -253,7 +253,7 @@ public class ParameterizedBuiltinCPFileInstruction extends ParameterizedBuiltinC
InputInfo iiOld = metadata.getInputInfo();
MatrixCharacteristics mc = new MatrixCharacteristics( rows, cols, mcOld.getRowsPerBlock(),
mcOld.getColsPerBlock(), mcOld.getNonZeros());
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,oiOld,iiOld);
+ MetaDataFormat meta = new MetaDataFormat(mc,oiOld,iiOld);
moNew.setMetaData( meta );
return moNew;
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java
index 43dd327..fb80deb 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java
@@ -35,7 +35,7 @@ import org.apache.sysml.runtime.instructions.cp.CPOperand;
import org.apache.sysml.runtime.instructions.spark.utils.FrameRDDConverterUtils;
import org.apache.sysml.runtime.instructions.spark.utils.RDDConverterUtils;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
@@ -94,7 +94,7 @@ public class CSVReblockSPInstruction extends UnarySPInstruction {
//sanity check input info
CacheableData<?> obj = sec.getCacheableData(input1.getName());
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) obj.getMetaData();
+ MetaDataFormat iimd = (MetaDataFormat) obj.getMetaData();
if (iimd.getInputInfo() != InputInfo.CSVInputInfo) {
throw new DMLRuntimeException("The given InputInfo is not implemented for "
+ "CSVReblockSPInstruction:" + iimd.getInputInfo());
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/instructions/spark/ConvolutionSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/ConvolutionSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/ConvolutionSPInstruction.java
index 4ea4f0b..874371d 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/ConvolutionSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/ConvolutionSPInstruction.java
@@ -37,7 +37,7 @@ import org.apache.sysml.runtime.instructions.spark.data.LazyIterableIterator;
import org.apache.sysml.runtime.instructions.spark.functions.ExtractBlockForBinaryReblock;
import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.ConvolutionParameters;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.LibMatrixDNN;
@@ -291,7 +291,7 @@ public class ConvolutionSPInstruction extends UnarySPInstruction {
throw new DMLRuntimeException("The current operator doesnot support large outputs.");
}
sec.setMetaData(output.getName(),
- new MatrixFormatMetaData(new MatrixCharacteristics(mcRdd.getRows(), numCols, numRowsPerBlock, (int)numCols, nnz), OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo));
+ new MetaDataFormat(new MatrixCharacteristics(mcRdd.getRows(), numCols, numRowsPerBlock, (int)numCols, nnz), OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo));
}
else {
throw new DMLRuntimeException("Not implemented: " + instOpcode);
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/instructions/spark/ReblockSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/ReblockSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/ReblockSPInstruction.java
index 0839c5b..d529084 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/ReblockSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/ReblockSPInstruction.java
@@ -37,7 +37,7 @@ import org.apache.sysml.runtime.instructions.spark.utils.FrameRDDConverterUtils;
import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
import org.apache.sysml.runtime.instructions.spark.utils.RDDConverterUtils;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.InputInfo;
@@ -92,7 +92,7 @@ public class ReblockSPInstruction extends UnarySPInstruction {
mcOut.set(mc.getRows(), mc.getCols(), brlen, bclen, mc.getNonZeros());
//get the source format form the meta data
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) obj.getMetaData();
+ MetaDataFormat iimd = (MetaDataFormat) obj.getMetaData();
if(iimd == null)
throw new DMLRuntimeException("Error: Metadata not found");
InputInfo iinfo = iimd.getInputInfo();