You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/11/04 01:58:46 UTC
[1/5] systemml git commit: [MINOR] Performance createvar,
rmvar and cleanup meta data handling
Repository: systemml
Updated Branches:
refs/heads/master f76f2138a -> 820b53046
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/matrix/GMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/GMR.java b/src/main/java/org/apache/sysml/runtime/matrix/GMR.java
index f0c214e..ae9cfce 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/GMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/GMR.java
@@ -47,7 +47,6 @@ import org.apache.sysml.runtime.instructions.mr.MRInstruction;
import org.apache.sysml.runtime.instructions.mr.PickByCountInstruction;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
-import org.apache.sysml.runtime.matrix.data.NumItemsByEachReducerMetaData;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
import org.apache.sysml.runtime.matrix.data.TaggedMatrixBlock;
import org.apache.sysml.runtime.matrix.data.TaggedMatrixPackedCell;
@@ -136,13 +135,13 @@ public class GMR
PickFromCompactInputFormat.setKeyValueClasses(job, (Class<? extends WritableComparable>) inputInfos[ins.input1].inputKeyClass,
inputInfos[ins.input1].inputValueClass);
job.setInputFormat(PickFromCompactInputFormat.class);
- PickFromCompactInputFormat.setZeroValues(job, (NumItemsByEachReducerMetaData)inputInfos[ins.input1].metadata);
+ PickFromCompactInputFormat.setZeroValues(job, (MetaDataNumItemsByEachReducer)inputInfos[ins.input1].metadata);
if(ins.isValuePick)
{
double[] probs=MapReduceTool.readColumnVectorFromHDFS(inputs[ins.input2], inputInfos[ins.input2], rlens[ins.input2],
clens[ins.input2], brlens[ins.input2], bclens[ins.input2]);
- PickFromCompactInputFormat.setPickRecordsInEachPartFile(job, (NumItemsByEachReducerMetaData) inputInfos[ins.input1].metadata, probs);
+ PickFromCompactInputFormat.setPickRecordsInEachPartFile(job, (MetaDataNumItemsByEachReducer) inputInfos[ins.input1].metadata, probs);
realinputs=new String[inputs.length-1];
realinputInfos=new InputInfo[inputs.length-1];
@@ -179,8 +178,8 @@ public class GMR
}else
{
//PickFromCompactInputFormat.setPickRecordsInEachPartFile(job, (NumItemsByEachReducerMetaData) inputInfos[ins.input1].metadata, ins.cst, 1-ins.cst);
- PickFromCompactInputFormat.setRangePickPartFiles(job, (NumItemsByEachReducerMetaData) inputInfos[ins.input1].metadata, ins.cst, 1-ins.cst);
- realrlens[ins.input1]=UtilFunctions.getLengthForInterQuantile((NumItemsByEachReducerMetaData)inputInfos[ins.input1].metadata, ins.cst);
+ PickFromCompactInputFormat.setRangePickPartFiles(job, (MetaDataNumItemsByEachReducer) inputInfos[ins.input1].metadata, ins.cst, 1-ins.cst);
+ realrlens[ins.input1]=UtilFunctions.getLengthForInterQuantile((MetaDataNumItemsByEachReducer)inputInfos[ins.input1].metadata, ins.cst);
realclens[ins.input1]=clens[ins.input1];
realbrlens[ins.input1]=1;
realbclens[ins.input1]=1;
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/matrix/JobReturn.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/JobReturn.java b/src/main/java/org/apache/sysml/runtime/matrix/JobReturn.java
index a2f3f8d..3d846a7 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/JobReturn.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/JobReturn.java
@@ -21,7 +21,6 @@
package org.apache.sysml.runtime.matrix;
import org.apache.sysml.runtime.DMLRuntimeException;
-import org.apache.sysml.runtime.matrix.data.NumItemsByEachReducerMetaData;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
@@ -39,31 +38,31 @@ public class JobReturn
public JobReturn(MatrixCharacteristics[] sts, boolean success) {
successful = success;
- metadata = new MatrixDimensionsMetaData[sts.length];
+ metadata = new MetaData[sts.length];
for (int i = 0; i < sts.length; i++) {
- metadata[i] = new MatrixDimensionsMetaData(sts[i]);
+ metadata[i] = new MetaData(sts[i]);
}
}
public JobReturn(MatrixCharacteristics[] sts, OutputInfo[] infos,
boolean success) throws DMLRuntimeException {
successful = success;
- metadata = new MatrixFormatMetaData[sts.length];
+ metadata = new MetaDataFormat[sts.length];
for (int i = 0; i < sts.length; i++) {
- metadata[i] = new MatrixFormatMetaData(sts[i], infos[i], OutputInfo.getMatchingInputInfo(infos[i]));
+ metadata[i] = new MetaDataFormat(sts[i], infos[i], OutputInfo.getMatchingInputInfo(infos[i]));
}
}
public JobReturn(MatrixCharacteristics sts, OutputInfo info, boolean success) throws DMLRuntimeException {
successful = success;
- metadata = new MatrixFormatMetaData[1];
- metadata[0] = new MatrixFormatMetaData(sts, info, OutputInfo.getMatchingInputInfo(info));
+ metadata = new MetaDataFormat[1];
+ metadata[0] = new MetaDataFormat(sts, info, OutputInfo.getMatchingInputInfo(info));
}
public JobReturn(MatrixCharacteristics mc, long[] items, int partition0, long number0s, boolean success) {
successful = success;
- metadata = new NumItemsByEachReducerMetaData[1];
- metadata[0] = new NumItemsByEachReducerMetaData(mc, items, partition0, number0s);
+ metadata = new MetaDataNumItemsByEachReducer[1];
+ metadata[0] = new MetaDataNumItemsByEachReducer(mc, items, partition0, number0s);
}
public boolean checkReturnStatus() throws DMLRuntimeException {
@@ -79,14 +78,4 @@ public class JobReturn
public MetaData getMetaData(int i) {
return metadata[i];
}
-
- /*
- * Since MatrixCharacteristics is the most common type of metadata, we
- * define a method to extract this information for a given output index.
- */
- public MatrixCharacteristics getMatrixCharacteristics(int i) {
- return ((MatrixDimensionsMetaData) metadata[i])
- .getMatrixCharacteristics();
- }
-
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java b/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java
index cb7ca57..a0e81ed 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java
@@ -21,6 +21,7 @@
package org.apache.sysml.runtime.matrix;
import java.io.Serializable;
+import java.util.Arrays;
import java.util.HashMap;
import org.apache.sysml.lops.MMTSJ.MMTSJType;
@@ -409,24 +410,20 @@ public class MatrixCharacteristics implements Serializable
}
@Override
- public boolean equals (Object anObject)
- {
- if (anObject instanceof MatrixCharacteristics)
- {
- MatrixCharacteristics mc = (MatrixCharacteristics) anObject;
- return ((numRows == mc.numRows) &&
- (numColumns == mc.numColumns) &&
- (numRowsPerBlock == mc.numRowsPerBlock) &&
- (numColumnsPerBlock == mc.numColumnsPerBlock) &&
- (nonZero == mc.nonZero)) ;
- }
- else
+ public boolean equals (Object anObject) {
+ if( !(anObject instanceof MatrixCharacteristics) )
return false;
+ MatrixCharacteristics mc = (MatrixCharacteristics) anObject;
+ return ((numRows == mc.numRows)
+ && (numColumns == mc.numColumns)
+ && (numRowsPerBlock == mc.numRowsPerBlock)
+ && (numColumnsPerBlock == mc.numColumnsPerBlock)
+ && (nonZero == mc.nonZero));
}
@Override
- public int hashCode()
- {
- return super.hashCode();
+ public int hashCode() {
+ return Arrays.hashCode(new long[]{numRows,numColumns,
+ numRowsPerBlock,numColumnsPerBlock,nonZero});
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/matrix/MatrixDimensionsMetaData.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/MatrixDimensionsMetaData.java b/src/main/java/org/apache/sysml/runtime/matrix/MatrixDimensionsMetaData.java
deleted file mode 100644
index a9bce8a..0000000
--- a/src/main/java/org/apache/sysml/runtime/matrix/MatrixDimensionsMetaData.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-package org.apache.sysml.runtime.matrix;
-
-public class MatrixDimensionsMetaData extends MetaData
-{
- protected MatrixCharacteristics matchar;
-
- public MatrixDimensionsMetaData() {
- matchar = null;
- }
-
- public MatrixDimensionsMetaData(MatrixCharacteristics mc) {
- matchar = mc;
- }
-
- public MatrixCharacteristics getMatrixCharacteristics() {
- return matchar;
- }
-
- public void setMatrixCharacteristics(MatrixCharacteristics mc) {
- matchar = mc;
- }
-
- @Override
- public boolean equals (Object anObject)
- {
- if (anObject instanceof MatrixDimensionsMetaData)
- {
- MatrixDimensionsMetaData md = (MatrixDimensionsMetaData) anObject;
- return (matchar.equals (md.matchar));
- }
- else
- return false;
- }
-
- @Override
- public int hashCode()
- {
- //use identity hash code
- return super.hashCode();
- }
-
- @Override
- public String toString() {
- return "[rows = " + matchar.getRows() +
- ", cols = " + matchar.getCols() +
- ", rpb = " + matchar.getRowsPerBlock() +
- ", cpb = " + matchar.getColsPerBlock() + "]";
- }
-
- @Override
- public Object clone()
- {
- MatrixCharacteristics mc = new MatrixCharacteristics(matchar);
- MatrixDimensionsMetaData ret = new MatrixDimensionsMetaData(mc);
-
- return ret;
- }
-}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/matrix/MatrixFormatMetaData.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/MatrixFormatMetaData.java b/src/main/java/org/apache/sysml/runtime/matrix/MatrixFormatMetaData.java
deleted file mode 100644
index 557e3eb..0000000
--- a/src/main/java/org/apache/sysml/runtime/matrix/MatrixFormatMetaData.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-package org.apache.sysml.runtime.matrix;
-
-import org.apache.sysml.runtime.matrix.data.InputInfo;
-import org.apache.sysml.runtime.matrix.data.OutputInfo;
-
-public class MatrixFormatMetaData extends MatrixDimensionsMetaData
-{
- private InputInfo iinfo;
- private OutputInfo oinfo;
-
- public MatrixFormatMetaData(MatrixCharacteristics mc, OutputInfo oinfo_, InputInfo iinfo_ ) {
- super(mc);
- oinfo = oinfo_;
- iinfo = iinfo_;
- }
-
- public InputInfo getInputInfo() {
- return iinfo;
- }
-
- public OutputInfo getOutputInfo() {
- return oinfo;
- }
-
- @Override
- public Object clone()
- {
- MatrixCharacteristics mc = new MatrixCharacteristics(matchar);
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc, oinfo, iinfo);
-
- return meta;
- }
-}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/matrix/MetaData.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/MetaData.java b/src/main/java/org/apache/sysml/runtime/matrix/MetaData.java
index 09b5c78..bb15c78 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/MetaData.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/MetaData.java
@@ -17,30 +17,44 @@
* under the License.
*/
-
package org.apache.sysml.runtime.matrix;
/**
* Class to store metadata associated with a file (e.g., a matrix) on disk.
- * This class must be extended to associate specific information with the file.
*
*/
-
-public abstract class MetaData
+public class MetaData
{
- @Override
- public abstract boolean equals (Object anObject);
+ protected final MatrixCharacteristics _mc;
+
+ public MetaData(MatrixCharacteristics mc) {
+ _mc = mc;
+ }
+
+ public MatrixCharacteristics getMatrixCharacteristics() {
+ return _mc;
+ }
@Override
- public int hashCode()
- {
- //use identity hash code
- return super.hashCode();
+ public boolean equals (Object anObject) {
+ if( !(anObject instanceof MetaData) )
+ return false;
+ MetaData that = (MetaData)anObject;
+ return _mc.equals(that._mc);
}
@Override
- public abstract String toString();
+ public int hashCode() {
+ return _mc.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return _mc.toString();
+ }
- @Override
- public abstract Object clone();
+ @Override
+ public Object clone() {
+ return new MetaData(new MatrixCharacteristics(_mc));
+ }
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/matrix/MetaDataFormat.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/MetaDataFormat.java b/src/main/java/org/apache/sysml/runtime/matrix/MetaDataFormat.java
new file mode 100644
index 0000000..ad033e1
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/matrix/MetaDataFormat.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+package org.apache.sysml.runtime.matrix;
+
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+
+public class MetaDataFormat extends MetaData
+{
+ private final InputInfo iinfo;
+ private final OutputInfo oinfo;
+
+ public MetaDataFormat(MatrixCharacteristics mc, OutputInfo oinfo, InputInfo iinfo ) {
+ super(mc);
+ this.oinfo = oinfo;
+ this.iinfo = iinfo;
+ }
+
+ public InputInfo getInputInfo() {
+ return iinfo;
+ }
+
+ public OutputInfo getOutputInfo() {
+ return oinfo;
+ }
+
+ @Override
+ public Object clone() {
+ return new MetaDataFormat(new MatrixCharacteristics(_mc), oinfo, iinfo);
+ }
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/matrix/MetaDataNumItemsByEachReducer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/MetaDataNumItemsByEachReducer.java b/src/main/java/org/apache/sysml/runtime/matrix/MetaDataNumItemsByEachReducer.java
new file mode 100644
index 0000000..d728c10
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/matrix/MetaDataNumItemsByEachReducer.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+package org.apache.sysml.runtime.matrix;
+
+public class MetaDataNumItemsByEachReducer extends MetaData
+{
+ private long[] numItems=null;
+ private int partitionOfZero=-1;
+ private long numberOfZero=0;
+
+ public MetaDataNumItemsByEachReducer(MatrixCharacteristics mc, long[] nums, int part0, long num0) {
+ super(mc);
+ numItems=nums;
+ partitionOfZero=part0;
+ numberOfZero=num0;
+ }
+
+ public long[] getNumItemsArray() {
+ return numItems;
+ }
+
+ public int getPartitionOfZero() {
+ return partitionOfZero;
+ }
+
+ public long getNumberOfZero() {
+ return numberOfZero;
+ }
+
+ @Override
+ public Object clone() {
+ return new MetaDataNumItemsByEachReducer(
+ new MatrixCharacteristics(_mc), numItems, partitionOfZero, numberOfZero);
+ }
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/matrix/data/NumItemsByEachReducerMetaData.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/NumItemsByEachReducerMetaData.java b/src/main/java/org/apache/sysml/runtime/matrix/data/NumItemsByEachReducerMetaData.java
deleted file mode 100644
index ac82403..0000000
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/NumItemsByEachReducerMetaData.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-package org.apache.sysml.runtime.matrix.data;
-
-import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixDimensionsMetaData;
-
-public class NumItemsByEachReducerMetaData extends MatrixDimensionsMetaData
-{
-
- private long[] numItems=null;
- private int partitionOfZero=-1;
- private long numberOfZero=0;
-
- public NumItemsByEachReducerMetaData(MatrixCharacteristics mc, long[] nums, int part0, long num0)
- {
- super(mc);
- numItems=nums;
- partitionOfZero=part0;
- numberOfZero=num0;
- }
-
- public long[] getNumItemsArray()
- {
- return numItems;
- }
-
- public int getPartitionOfZero()
- {
- return partitionOfZero;
- }
-
- public long getNumberOfZero()
- {
- return numberOfZero;
- }
-
- @Override
- public Object clone()
- {
- MatrixCharacteristics mc = new MatrixCharacteristics(matchar);
- NumItemsByEachReducerMetaData ret = new NumItemsByEachReducerMetaData(mc, numItems, partitionOfZero, numberOfZero);
-
- return ret;
- }
-}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/matrix/sort/PickFromCompactInputFormat.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/sort/PickFromCompactInputFormat.java b/src/main/java/org/apache/sysml/runtime/matrix/sort/PickFromCompactInputFormat.java
index a615741..903c3bd 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/sort/PickFromCompactInputFormat.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/sort/PickFromCompactInputFormat.java
@@ -41,9 +41,9 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
import org.apache.sysml.runtime.io.IOUtilFunctions;
+import org.apache.sysml.runtime.matrix.MetaDataNumItemsByEachReducer;
import org.apache.sysml.runtime.matrix.data.MatrixCell;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
-import org.apache.sysml.runtime.matrix.data.NumItemsByEachReducerMetaData;
import org.apache.sysml.runtime.matrix.data.Pair;
//key class to read has to be DoubleWritable
@@ -62,7 +62,7 @@ public class PickFromCompactInputFormat extends FileInputFormat<MatrixIndexes, M
return false;
}
- public static void setZeroValues(JobConf job, NumItemsByEachReducerMetaData metadata)
+ public static void setZeroValues(JobConf job, MetaDataNumItemsByEachReducer metadata)
{
job.setInt(PARTITION_OF_ZERO, metadata.getPartitionOfZero());
job.setLong(NUMBER_OF_ZERO, metadata.getNumberOfZero());
@@ -141,7 +141,7 @@ public class PickFromCompactInputFormat extends FileInputFormat<MatrixIndexes, M
}
}
- public static Set<Integer> setPickRecordsInEachPartFile(JobConf job, NumItemsByEachReducerMetaData metadata, double[] probs)
+ public static Set<Integer> setPickRecordsInEachPartFile(JobConf job, MetaDataNumItemsByEachReducer metadata, double[] probs)
{
HashMap<Integer, ArrayList<Pair<Integer, Integer>>> posMap = new HashMap<>();
getPointsInEachPartFile(metadata.getNumItemsArray(), probs, posMap);
@@ -155,7 +155,7 @@ public class PickFromCompactInputFormat extends FileInputFormat<MatrixIndexes, M
return posMap.keySet();
}
- public static void setRangePickPartFiles(JobConf job, NumItemsByEachReducerMetaData metadata, double lbound, double ubound) {
+ public static void setRangePickPartFiles(JobConf job, MetaDataNumItemsByEachReducer metadata, double lbound, double ubound) {
if(lbound<0 || lbound > 1 || ubound <0 || ubound >1 || lbound >= ubound ) {
throw new RuntimeException("Invalid ranges for range pick: [" + lbound + "," + ubound + "]");
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java b/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java
index e8f206a..eb2b141 100644
--- a/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java
+++ b/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java
@@ -51,11 +51,11 @@ import org.apache.sysml.runtime.io.IOUtilFunctions;
import org.apache.sysml.runtime.io.MatrixReader;
import org.apache.sysml.runtime.io.MatrixReaderFactory;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.MetaDataNumItemsByEachReducer;
import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
import org.apache.sysml.runtime.matrix.data.FileFormatProperties;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
-import org.apache.sysml.runtime.matrix.data.NumItemsByEachReducerMetaData;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
import org.apache.sysml.runtime.matrix.mapred.MRConfigurationNames;
import org.apache.sysml.runtime.matrix.sort.ReadWithZeros;
@@ -526,7 +526,7 @@ public class MapReduceTool
return DataConverter.convertToDoubleVector(mb, false);
}
- public static double median(String dir, NumItemsByEachReducerMetaData metadata) throws IOException {
+ public static double median(String dir, MetaDataNumItemsByEachReducer metadata) throws IOException {
long[] counts=metadata.getNumItemsArray();
long[] ranges=new long[counts.length];
ranges[0]=counts[0];
@@ -539,11 +539,11 @@ public class MapReduceTool
}
- public static double pickValue(String dir, NumItemsByEachReducerMetaData metadata, double p) throws IOException {
+ public static double pickValue(String dir, MetaDataNumItemsByEachReducer metadata, double p) throws IOException {
return pickValueWeight(dir, metadata, p, false)[0];
}
- public static double[] pickValueWeight(String dir, NumItemsByEachReducerMetaData metadata, double p, boolean average)
+ public static double[] pickValueWeight(String dir, MetaDataNumItemsByEachReducer metadata, double p, boolean average)
throws IOException
{
long[] counts=metadata.getNumItemsArray();
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java b/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
index 5644ac6..199108b 100644
--- a/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
+++ b/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
@@ -27,9 +27,9 @@ import java.util.Set;
import org.apache.commons.lang.ArrayUtils;
import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.matrix.MetaDataNumItemsByEachReducer;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
-import org.apache.sysml.runtime.matrix.data.NumItemsByEachReducerMetaData;
import org.apache.sysml.runtime.matrix.data.Pair;
import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
@@ -242,7 +242,7 @@ public class UtilFunctions
return new IndexRange(iRowStart, iRowEnd, iColStart, iColEnd);
}
- public static long getTotalLength(NumItemsByEachReducerMetaData metadata) {
+ public static long getTotalLength(MetaDataNumItemsByEachReducer metadata) {
long[] counts=metadata.getNumItemsArray();
long total=0;
for(long count: counts)
@@ -250,7 +250,7 @@ public class UtilFunctions
return total;
}
- public static long getLengthForInterQuantile(NumItemsByEachReducerMetaData metadata, double p)
+ public static long getLengthForInterQuantile(MetaDataNumItemsByEachReducer metadata, double p)
{
long total = UtilFunctions.getTotalLength(metadata);
long lpos=(long)Math.ceil(total*p);//lower bound is inclusive
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/udf/Matrix.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/Matrix.java b/src/main/java/org/apache/sysml/udf/Matrix.java
index 1e6dfbd..5da92e3 100644
--- a/src/main/java/org/apache/sysml/udf/Matrix.java
+++ b/src/main/java/org/apache/sysml/udf/Matrix.java
@@ -29,7 +29,7 @@ import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
import org.apache.sysml.runtime.io.MatrixReader;
import org.apache.sysml.runtime.io.MatrixReaderFactory;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
@@ -216,7 +216,7 @@ public class Matrix extends FunctionParameter
int cblen = ConfigurationManager.getBlocksize();
MatrixCharacteristics mc = new MatrixCharacteristics(_rows, _cols, rblen, cblen, nnz);
- MatrixFormatMetaData mfmd = new MatrixFormatMetaData(mc, oinfo, iinfo);
+ MetaDataFormat mfmd = new MetaDataFormat(mc, oinfo, iinfo);
try
{
_mo = new MatrixObject(Expression.ValueType.DOUBLE, _filePath, mfmd);
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java b/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java
index 0378924..c5cabbe 100644
--- a/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java
+++ b/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java
@@ -57,7 +57,7 @@ import org.apache.sysml.runtime.controlprogram.parfor.stat.Timing;
import org.apache.sysml.runtime.instructions.Instruction;
import org.apache.sysml.runtime.instructions.MRJobInstruction;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixDimensionsMetaData;
+import org.apache.sysml.runtime.matrix.MetaData;
import org.apache.sysml.yarn.DMLYarnClient;
import org.apache.sysml.yarn.ropt.YarnOptimizerUtils.GridEnumType;
@@ -440,8 +440,7 @@ public class ResourceOptimizer
String varname = hop.getName();
MatrixCharacteristics mc = new MatrixCharacteristics(hop.getDim1(), hop.getDim2(),
(int)hop.getRowsInBlock(), (int)hop.getColsInBlock(), hop.getNnz());
- MatrixDimensionsMetaData md = new MatrixDimensionsMetaData(mc);
- MatrixObject mo = new MatrixObject(ValueType.DOUBLE, "/tmp", md);
+ MatrixObject mo = new MatrixObject(ValueType.DOUBLE, "/tmp", new MetaData(mc));
vars.put(varname, mo);
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameEvictionTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameEvictionTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameEvictionTest.java
index 7d0abfa..e3c8281 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameEvictionTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameEvictionTest.java
@@ -26,7 +26,7 @@ import org.apache.sysml.runtime.controlprogram.caching.CacheableData;
import org.apache.sysml.runtime.controlprogram.caching.FrameObject;
import org.apache.sysml.runtime.controlprogram.caching.LazyWriteBuffer;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
@@ -202,7 +202,7 @@ public class FrameEvictionTest extends AutomatedTestBase
//create frame object
MatrixCharacteristics mc = new MatrixCharacteristics(rows, schema.length, -1, -1, -1);
- MatrixFormatMetaData meta = new MatrixFormatMetaData (mc,
+ MetaDataFormat meta = new MetaDataFormat (mc,
OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
FrameObject fo = new FrameObject("fA", meta, schema);
fo.acquireModify(fA);
[5/5] systemml git commit: [SYSTEMML-1984] Fix special cases of JMLC
prepare/execute sequences
Posted by mb...@apache.org.
[SYSTEMML-1984] Fix special cases of JMLC prepare/execute sequences
This patch fixes issues with special cases of prepare and execute
sequences on the same connection, which currently fail due to the
cleanup of thread-local configurations after execute.
Furthermore, this also adds the existing JMLC tests from building the
lite jar to our testsuite in order to avoid regressions. This also
includes a minor fix of these tests regarding the recently modified
L2SVM predict script.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/820b5304
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/820b5304
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/820b5304
Branch: refs/heads/master
Commit: 820b5304672a81093787670c70b1c083daab446d
Parents: 4f60ded
Author: Matthias Boehm <mb...@gmail.com>
Authored: Fri Nov 3 18:59:00 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Fri Nov 3 18:59:30 2017 -0700
----------------------------------------------------------------------
.../org/apache/sysml/api/jmlc/Connection.java | 10 ++-
.../sysml/utils/lite/BuildLiteExecution.java | 4 +-
.../functions/jmlc/BuildLiteJarTest.java | 87 ++++++++++++++++++++
.../functions/jmlc/ZPackageSuite.java | 1 +
4 files changed, 97 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/820b5304/src/main/java/org/apache/sysml/api/jmlc/Connection.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/jmlc/Connection.java b/src/main/java/org/apache/sysml/api/jmlc/Connection.java
index 4caa86c..e96e0aa 100644
--- a/src/main/java/org/apache/sysml/api/jmlc/Connection.java
+++ b/src/main/java/org/apache/sysml/api/jmlc/Connection.java
@@ -131,7 +131,7 @@ public class Connection implements Closeable
//create default configuration
_dmlconf = new DMLConfig();
- //set thread-local configurations for compilation
+ //set thread-local configurations for compilation and read
ConfigurationManager.setLocalConfig(_dmlconf);
ConfigurationManager.setLocalConfig(_cconf);
}
@@ -150,7 +150,7 @@ public class Connection implements Closeable
for( ConfigType configType : configs )
_cconf.set(configType, true);
- //set thread-local configurations for compilation
+ //set thread-local configurations for compilation and read
ConfigurationManager.setLocalConfig(_dmlconf);
ConfigurationManager.setLocalConfig(_cconf);
}
@@ -186,7 +186,7 @@ public class Connection implements Closeable
throws DMLException
{
DMLScript.SCRIPT_TYPE = parsePyDML ? ScriptType.PYDML : ScriptType.DML;
-
+
//check for valid names of passed arguments
String[] invalidArgs = args.keySet().stream()
.filter(k -> k==null || !k.startsWith("$")).toArray(String[]::new);
@@ -202,6 +202,10 @@ public class Connection implements Closeable
//simplified compilation chain
Program rtprog = null;
try {
+ //set thread-local configurations for compilation
+ ConfigurationManager.setLocalConfig(_dmlconf);
+ ConfigurationManager.setLocalConfig(_cconf);
+
//parsing
ParserWrapper parser = ParserFactory.createParser(parsePyDML ? ScriptType.PYDML : ScriptType.DML);
DMLProgram prog = parser.parse(null, script, args);
http://git-wip-us.apache.org/repos/asf/systemml/blob/820b5304/src/main/java/org/apache/sysml/utils/lite/BuildLiteExecution.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/utils/lite/BuildLiteExecution.java b/src/main/java/org/apache/sysml/utils/lite/BuildLiteExecution.java
index b2b2574..4ce6f33 100644
--- a/src/main/java/org/apache/sysml/utils/lite/BuildLiteExecution.java
+++ b/src/main/java/org/apache/sysml/utils/lite/BuildLiteExecution.java
@@ -330,7 +330,7 @@ public class BuildLiteExecution {
m.put("$confusion", "temp/2.csv");
m.put("$scores", "temp/3.csv");
- PreparedScript l2svmPredict = conn.prepareScript(s, m, new String[] { "X", "y", "w", "fmt" },
+ PreparedScript l2svmPredict = conn.prepareScript(s, m, new String[] { "X", "Y", "w", "fmt" },
new String[] { "scores", "confusion_mat" }, false);
double[][] testData = new double[150][3];
@@ -349,7 +349,7 @@ public class BuildLiteExecution {
double[] row = new double[] { one };
testLabels[i] = row;
}
- l2svmPredict.setMatrix("y", testLabels);
+ l2svmPredict.setMatrix("Y", testLabels);
l2svmPredict.setMatrix("w", model);
http://git-wip-us.apache.org/repos/asf/systemml/blob/820b5304/src/test/java/org/apache/sysml/test/integration/functions/jmlc/BuildLiteJarTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/jmlc/BuildLiteJarTest.java b/src/test/java/org/apache/sysml/test/integration/functions/jmlc/BuildLiteJarTest.java
new file mode 100644
index 0000000..24c400f
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/jmlc/BuildLiteJarTest.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.jmlc;
+
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.utils.lite.BuildLiteExecution;
+import org.junit.Test;
+
+public class BuildLiteJarTest extends AutomatedTestBase
+{
+ @Override
+ public void setUp() {
+ //do nothing
+ }
+
+ @Test
+ public void testJMLCHelloWorld() throws Exception {
+ BuildLiteExecution.jmlcHelloWorld();
+ }
+
+ @Test
+ public void testJMLCScoringExample() throws Exception {
+ BuildLiteExecution.jmlcScoringExample();
+ }
+
+ @Test
+ public void testJMLCUnivariateStats() throws Exception {
+ BuildLiteExecution.jmlcUnivariateStatistics();
+ }
+
+ @Test
+ public void testJMLCWriteMatrix() throws Exception {
+ BuildLiteExecution.jmlcWriteMatrix();
+ }
+
+ @Test
+ public void testJMLCReadMatrix() throws Exception {
+ BuildLiteExecution.jmlcReadMatrix();
+ }
+
+ @Test
+ public void testJMLCBasics() throws Exception {
+ BuildLiteExecution.jmlcBasics();
+ }
+
+ @Test
+ public void testJMLCL2SVM() throws Exception {
+ BuildLiteExecution.jmlcL2SVM();
+ }
+
+ @Test
+ public void testJMLCLinReg() throws Exception {
+ BuildLiteExecution.jmlcLinReg();
+ }
+
+ @Test
+ public void testJMLCALS() throws Exception {
+ BuildLiteExecution.jmlcALS();
+ }
+
+ @Test
+ public void testJMLCKmeans() throws Exception {
+ BuildLiteExecution.jmlcKmeans();
+ }
+
+ @Test
+ public void testJMLCTests() throws Exception {
+ BuildLiteExecution.jmlcTests();
+ }
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/820b5304/src/test_suites/java/org/apache/sysml/test/integration/functions/jmlc/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/jmlc/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/jmlc/ZPackageSuite.java
index 87244aa..3d5c13a 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/jmlc/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/jmlc/ZPackageSuite.java
@@ -26,6 +26,7 @@ import org.junit.runners.Suite;
* won't run two of them at once. */
@RunWith(Suite.class)
@Suite.SuiteClasses({
+ BuildLiteJarTest.class,
FrameCastingTest.class,
FrameDecodeTest.class,
FrameEncodeTest.class,
[4/5] systemml git commit: [MINOR] Performance library UDF
rowClassMeet (sparse inputs, grouping)
Posted by mb...@apache.org.
[MINOR] Performance library UDF rowClassMeet (sparse inputs, grouping)
This patch makes some minor performance improvements to the existing
rowClassMeet UDF, by (1) using the codegen sparse side inputs for more
efficient, zero-copy access, and (2) a more time- and memory-efficient
hash grouping approach.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/4f60ded3
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/4f60ded3
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/4f60ded3
Branch: refs/heads/master
Commit: 4f60ded3543b0475f8e2bf987febc33e66e2652e
Parents: 14c410c
Author: Matthias Boehm <mb...@gmail.com>
Authored: Fri Nov 3 18:04:29 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Fri Nov 3 18:59:29 2017 -0700
----------------------------------------------------------------------
.../sysml/runtime/codegen/CodegenUtils.java | 10 +
.../sysml/runtime/codegen/SpoofOperator.java | 8 +
.../org/apache/sysml/udf/lib/RowClassMeet.java | 206 ++++++++-----------
3 files changed, 102 insertions(+), 122 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/4f60ded3/src/main/java/org/apache/sysml/runtime/codegen/CodegenUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/codegen/CodegenUtils.java b/src/main/java/org/apache/sysml/runtime/codegen/CodegenUtils.java
index b4acb71..8b3ea5f 100644
--- a/src/main/java/org/apache/sysml/runtime/codegen/CodegenUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/codegen/CodegenUtils.java
@@ -46,7 +46,10 @@ import org.apache.sysml.api.DMLScript;
import org.apache.sysml.hops.codegen.SpoofCompiler;
import org.apache.sysml.hops.codegen.SpoofCompiler.CompilerType;
import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.codegen.SpoofOperator.SideInput;
+import org.apache.sysml.runtime.codegen.SpoofOperator.SideInputSparseCell;
import org.apache.sysml.runtime.io.IOUtilFunctions;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.util.LocalFileUtils;
import org.apache.sysml.utils.Statistics;
import org.codehaus.janino.SimpleCompiler;
@@ -152,6 +155,13 @@ public class CodegenUtils
return ret;
}
+ public static SideInput createSideInput(MatrixBlock in) {
+ SideInput ret = (in.isInSparseFormat() || !in.isAllocated()) ?
+ new SideInput(null, in, in.getNumColumns()) :
+ new SideInput(in.getDenseBlock(), null, in.getNumColumns());
+ return (ret.mdat != null) ? new SideInputSparseCell(ret) : ret;
+ }
+
////////////////////////////
//JANINO-specific methods (used for spark environments)
http://git-wip-us.apache.org/repos/asf/systemml/blob/4f60ded3/src/main/java/org/apache/sysml/runtime/codegen/SpoofOperator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/codegen/SpoofOperator.java b/src/main/java/org/apache/sysml/runtime/codegen/SpoofOperator.java
index 21fdd35..4505c3e 100644
--- a/src/main/java/org/apache/sysml/runtime/codegen/SpoofOperator.java
+++ b/src/main/java/org/apache/sysml/runtime/codegen/SpoofOperator.java
@@ -249,6 +249,10 @@ public abstract class SpoofOperator implements Serializable
public double[] values(int r) {
return ddat;
}
+ public double getValue(int r, int c) {
+ return SpoofOperator.getValue(this, clen, r, c);
+ }
+ public void reset() {}
}
public static class SideInputSparseRow extends SideInput {
@@ -315,5 +319,9 @@ public abstract class SpoofOperator implements Serializable
return (currColPos < currLen && indexes[currColPos]==colIndex) ?
values[currColPos] : 0;
}
+ @Override
+ public void reset() {
+ currColPos = 0;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/4f60ded3/src/main/java/org/apache/sysml/udf/lib/RowClassMeet.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/lib/RowClassMeet.java b/src/main/java/org/apache/sysml/udf/lib/RowClassMeet.java
index 5764d1e..273c7d8 100644
--- a/src/main/java/org/apache/sysml/udf/lib/RowClassMeet.java
+++ b/src/main/java/org/apache/sysml/udf/lib/RowClassMeet.java
@@ -19,24 +19,23 @@
package org.apache.sysml.udf.lib;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.Iterator;
+import java.util.HashMap;
import java.util.Map.Entry;
-import java.util.TreeMap;
import org.apache.sysml.runtime.DMLRuntimeException;
-import org.apache.sysml.runtime.controlprogram.caching.CacheException;
-import org.apache.sysml.runtime.matrix.data.IJV;
+import org.apache.sysml.runtime.codegen.CodegenUtils;
+import org.apache.sysml.runtime.codegen.SpoofOperator.SideInput;
+import org.apache.sysml.runtime.compress.utils.IntArrayList;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.UtilFunctions;
import org.apache.sysml.udf.FunctionParameter;
import org.apache.sysml.udf.Matrix;
import org.apache.sysml.udf.PackageFunction;
import org.apache.sysml.udf.Matrix.ValueType;
+
/**
* Performs following operation:
* # Computes the intersection ("meet") of equivalence classes for
@@ -73,9 +72,7 @@ public class RowClassMeet extends PackageFunction {
private static final long serialVersionUID = 1L;
private Matrix CMat, NMat;
- private MatrixBlock A, B, C, N;
- private int nr, nc;
-
+
@Override
public int getNumFunctionOutputs() {
return 2;
@@ -83,124 +80,75 @@ public class RowClassMeet extends PackageFunction {
@Override
public FunctionParameter getFunctionOutput(int pos) {
- if(pos == 0)
- return CMat;
- else if(pos == 1)
- return NMat;
- else
- throw new RuntimeException("RowClassMeet produces only one output");
- }
-
-
- public class ClassLabels {
- public double aVal;
- public double bVal;
- public ClassLabels(double aVal, double bVal) {
- this.aVal = aVal;
- this.bVal = bVal;
+ switch( pos ) {
+ case 0: return CMat;
+ case 1: return NMat;
+ default:
+ throw new RuntimeException("RowClassMeet produces only one output");
}
}
- public class ClassLabelComparator implements Comparator<ClassLabels> {
- Integer tmp1, tmp2;
- @Override
- public int compare(ClassLabels o1, ClassLabels o2) {
- if(o1.aVal != o2.aVal) {
- tmp1 = (int) o1.aVal;
- tmp2 = (int) o2.aVal;
- }
- else {
- tmp1 = (int) o1.bVal;
- tmp2 = (int) o2.bVal;
- }
- return tmp1.compareTo(tmp2);
- }
- }
-
- double [] getRow(MatrixBlock B, double [] bRow, int i) {
- if(B.getNumRows() == 1)
- i = 0;
- Arrays.fill(bRow, 0);
- if(B.isInSparseFormat()) {
- Iterator<IJV> iter = B.getSparseBlockIterator(i, i+1);
- while(iter.hasNext()) {
- IJV ijv = iter.next();
- bRow[ijv.getJ()] = ijv.getV();
- }
- }
- else {
- double [] denseBlk = B.getDenseBlock();
- if(denseBlk != null)
- System.arraycopy(denseBlk, i*B.getNumColumns(), bRow, 0, B.getNumColumns());
- }
- return bRow;
- }
-
@Override
public void execute() {
- try {
- A = ((Matrix) getFunctionInput(0)).getMatrixObject().acquireRead();
- B = ((Matrix) getFunctionInput(1)).getMatrixObject().acquireRead();
- nr = Math.max(A.getNumRows(), B.getNumRows());
- nc = Math.max(A.getNumColumns(), B.getNumColumns());
+ try
+ {
+ MatrixBlock A = ((Matrix) getFunctionInput(0)).getMatrixObject().acquireRead();
+ MatrixBlock B = ((Matrix) getFunctionInput(1)).getMatrixObject().acquireRead();
+ int nr = Math.max(A.getNumRows(), B.getNumRows());
+ int nc = Math.max(A.getNumColumns(), B.getNumColumns());
+ MatrixBlock C = new MatrixBlock(nr, nc, false).allocateBlock();
+ MatrixBlock N = new MatrixBlock(nr, nc, false).allocateBlock();
+ double[] dC = C.getDenseBlock();
+ double[] dN = N.getDenseBlock();
+ //wrap both A and B into side inputs for efficient sparse access
+ SideInput sB = CodegenUtils.createSideInput(B);
+ boolean mv = (B.getNumRows() == 1);
+ int numCols = Math.min(A.getNumColumns(), B.getNumColumns());
- double [] bRow = new double[B.getNumColumns()];
- CMat = new Matrix( createOutputFilePathAndName( "TMP" ), nr, nc, ValueType.Double );
- C = new MatrixBlock(nr, nc, false);
- C.allocateDenseBlock();
- NMat = new Matrix( createOutputFilePathAndName( "TMP" ), nr, nc, ValueType.Double );
- N = new MatrixBlock(nr, nc, false);
- N.allocateDenseBlock();
+ HashMap<ClassLabel, IntArrayList> classLabelMapping = new HashMap<>();
- double [] cBlk = C.getDenseBlock();
- double [] nBlk = N.getDenseBlock();
-
- if(B.getNumRows() == 1)
- getRow(B, bRow, 0);
-
- for(int i = 0; i < A.getNumRows(); i++) {
- if(B.getNumRows() != 1)
- getRow(B, bRow, i);
-
- // Create class labels
- TreeMap<ClassLabels, ArrayList<Integer>> classLabelMapping = new TreeMap<>(new ClassLabelComparator());
- if(A.isInSparseFormat()) {
- Iterator<IJV> iter = A.getSparseBlockIterator(i, i+1);
- while(iter.hasNext()) {
- IJV ijv = iter.next();
- int j = ijv.getJ();
- double aVal = ijv.getV();
- if(aVal != 0 && bRow[j] != 0) {
- ClassLabels key = new ClassLabels(aVal, bRow[j]);
+ for(int i=0, ai=0; i < A.getNumRows(); i++, ai+=A.getNumColumns()) {
+ classLabelMapping.clear(); sB.reset();
+ if( A.isInSparseFormat() ) {
+ if(A.getSparseBlock()==null || A.getSparseBlock().isEmpty(i))
+ continue;
+ int alen = A.getSparseBlock().size(i);
+ int apos = A.getSparseBlock().pos(i);
+ int[] aix = A.getSparseBlock().indexes(i);
+ double[] avals = A.getSparseBlock().values(i);
+ for(int k=apos; k<apos+alen; k++) {
+ if( aix[k] >= numCols ) break;
+ int bval = (int)sB.getValue(mv?0:i, aix[k]);
+ if( bval != 0 ) {
+ ClassLabel key = new ClassLabel((int)avals[k], bval);
if(!classLabelMapping.containsKey(key))
- classLabelMapping.put(key, new ArrayList<Integer>());
- classLabelMapping.get(key).add(j);
+ classLabelMapping.put(key, new IntArrayList());
+ classLabelMapping.get(key).appendValue(aix[k]);
}
}
}
else {
double [] denseBlk = A.getDenseBlock();
- if(denseBlk != null) {
- int offset = i*A.getNumColumns();
- for(int j = 0; j < A.getNumColumns(); j++) {
- double aVal = denseBlk[offset + j];
- if(aVal != 0 && bRow[j] != 0) {
- ClassLabels key = new ClassLabels(aVal, bRow[j]);
- if(!classLabelMapping.containsKey(key))
- classLabelMapping.put(key, new ArrayList<Integer>());
- classLabelMapping.get(key).add(j);
- }
+ if(denseBlk == null) break;
+ for(int j = 0; j < numCols; j++) {
+ int aVal = (int) denseBlk[ai+j];
+ int bVal = (int) sB.getValue(mv?0:i, j);
+ if(aVal != 0 && bVal != 0) {
+ ClassLabel key = new ClassLabel(aVal, bVal);
+ if(!classLabelMapping.containsKey(key))
+ classLabelMapping.put(key, new IntArrayList());
+ classLabelMapping.get(key).appendValue(j);
}
}
}
-
int labelID = 1;
- for(Entry<ClassLabels, ArrayList<Integer>> entry : classLabelMapping.entrySet()) {
- double nVal = entry.getValue().size();
- for(Integer j : entry.getValue()) {
- nBlk[i*nc + j] = nVal;
- cBlk[i*nc + j] = labelID;
+ for(Entry<ClassLabel, IntArrayList> entry : classLabelMapping.entrySet()) {
+ int nVal = entry.getValue().size();
+ int[] list = entry.getValue().extractValues();
+ for(int k=0, off=i*nc; k<nVal; k++) {
+ dN[off+list[k]] = nVal;
+ dC[off+list[k]] = labelID;
}
labelID++;
}
@@ -208,23 +156,37 @@ public class RowClassMeet extends PackageFunction {
((Matrix) getFunctionInput(0)).getMatrixObject().release();
((Matrix) getFunctionInput(1)).getMatrixObject().release();
- } catch (CacheException e) {
- throw new RuntimeException("Error while executing RowClassMeet", e);
- }
- try {
- C.recomputeNonZeros();
- C.examSparsity();
+ //prepare outputs
+ C.recomputeNonZeros(); C.examSparsity();
+ CMat = new Matrix( createOutputFilePathAndName( "TMP" ), nr, nc, ValueType.Double );
CMat.setMatrixDoubleArray(C, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
- N.recomputeNonZeros();
- N.examSparsity();
+ N.recomputeNonZeros(); N.examSparsity();
+ NMat = new Matrix( createOutputFilePathAndName( "TMP" ), nr, nc, ValueType.Double );
NMat.setMatrixDoubleArray(N, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
- } catch (DMLRuntimeException e) {
- throw new RuntimeException("Error while executing RowClassMeet", e);
- } catch (IOException e) {
+ }
+ catch (DMLRuntimeException | IOException e) {
throw new RuntimeException("Error while executing RowClassMeet", e);
}
}
-
+ private static class ClassLabel {
+ public int aVal;
+ public int bVal;
+ public ClassLabel(int aVal, int bVal) {
+ this.aVal = aVal;
+ this.bVal = bVal;
+ }
+ @Override
+ public int hashCode() {
+ return UtilFunctions.intHashCode(aVal, bVal);
+ }
+ @Override
+ public boolean equals(Object o) {
+ if( !(o instanceof ClassLabel) )
+ return false;
+ ClassLabel that = (ClassLabel) o;
+ return aVal == that.aVal && bVal == that.bVal;
+ }
+ }
}
[3/5] systemml git commit: [MINOR] Performance function invocation of
dml-bodied UDFs
Posted by mb...@apache.org.
[MINOR] Performance function invocation of dml-bodied UDFs
This patch slightly improved the function invocation performance of
dml-bodied UDFs from 452K/s to 521K/s.
Furthermore, this also includes a fix of the test for LinregCG over
compressed data.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/14c410ce
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/14c410ce
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/14c410ce
Branch: refs/heads/master
Commit: 14c410ce06f3a5c56d1bcb1ac509fab4a0711f5f
Parents: d7d312c
Author: Matthias Boehm <mb...@gmail.com>
Authored: Fri Nov 3 14:23:50 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Fri Nov 3 18:59:29 2017 -0700
----------------------------------------------------------------------
.../controlprogram/LocalVariableMap.java | 4 +
.../controlprogram/ParForProgramBlock.java | 4 +-
.../context/ExecutionContext.java | 41 +++---
.../cp/FunctionCallCPInstruction.java | 135 +++++++++----------
.../functions/compress/CompressedLinregCG.java | 19 ++-
5 files changed, 95 insertions(+), 108 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/14c410ce/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java b/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java
index 7ebe1a0..e894495 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java
@@ -63,6 +63,10 @@ public class LocalVariableMap implements Cloneable
return localMap.keySet();
}
+ public Set<Entry<String, Data>> entrySet() {
+ return localMap.entrySet();
+ }
+
/**
* Retrieves the data object given its name.
*
http://git-wip-us.apache.org/repos/asf/systemml/blob/14c410ce/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
index e2568cb..760ddff 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
@@ -633,7 +633,7 @@ public class ParForProgramBlock extends ForProgramBlock
//preserve shared input/result variables of cleanup
ArrayList<String> varList = ec.getVarList();
- HashMap<String, Boolean> varState = ec.pinVariables(varList);
+ boolean[] varState = ec.pinVariables(varList);
try
{
@@ -1329,7 +1329,7 @@ public class ParForProgramBlock extends ForProgramBlock
}
}
- private void cleanupSharedVariables( ExecutionContext ec, HashMap<String,Boolean> varState )
+ private void cleanupSharedVariables( ExecutionContext ec, boolean[] varState )
throws DMLRuntimeException
{
//TODO needs as precondition a systematic treatment of persistent read information.
http://git-wip-us.apache.org/repos/asf/systemml/blob/14c410ce/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
index 79a658d..ecb9629 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
@@ -20,7 +20,6 @@
package org.apache.sysml.runtime.controlprogram.context;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
import org.apache.commons.logging.Log;
@@ -151,6 +150,11 @@ public class ExecutionContext {
return _variables.get(name);
}
+ public Data getVariable(CPOperand operand) throws DMLRuntimeException {
+ return operand.getDataType().isScalar() ?
+ getScalarInput(operand) : getVariable(operand.getName());
+ }
+
public void setVariable(String name, Data val) {
_variables.put(name, val);
}
@@ -528,30 +532,25 @@ public class ExecutionContext {
* The function returns the OLD "clean up" state of matrix objects.
*
* @param varList variable list
- * @return map of old cleanup state of matrix objects
+ * @return indicator vector of old cleanup state of matrix objects
*/
- public HashMap<String,Boolean> pinVariables(ArrayList<String> varList)
+ public boolean[] pinVariables(ArrayList<String> varList)
{
//2-pass approach since multiple vars might refer to same matrix object
- HashMap<String, Boolean> varsState = new HashMap<>();
+ boolean[] varsState = new boolean[varList.size()];
//step 1) get current information
- for( String var : varList )
- {
- Data dat = _variables.get(var);
- if( dat instanceof MatrixObject ) {
- MatrixObject mo = (MatrixObject)dat;
- varsState.put( var, mo.isCleanupEnabled() );
- }
+ for( int i=0; i<varList.size(); i++ ) {
+ Data dat = _variables.get(varList.get(i));
+ if( dat instanceof MatrixObject )
+ varsState[i] = ((MatrixObject)dat).isCleanupEnabled();
}
//step 2) pin variables
- for( String var : varList ) {
- Data dat = _variables.get(var);
- if( dat instanceof MatrixObject ) {
- MatrixObject mo = (MatrixObject)dat;
- mo.enableCleanup(false);
- }
+ for( int i=0; i<varList.size(); i++ ) {
+ Data dat = _variables.get(varList.get(i));
+ if( dat instanceof MatrixObject )
+ ((MatrixObject)dat).enableCleanup(false);
}
return varsState;
@@ -573,11 +572,11 @@ public class ExecutionContext {
* @param varList variable list
* @param varsState variable state
*/
- public void unpinVariables(ArrayList<String> varList, HashMap<String,Boolean> varsState) {
- for( String var : varList) {
- Data dat = _variables.get(var);
+ public void unpinVariables(ArrayList<String> varList, boolean[] varsState) {
+ for( int i=0; i<varList.size(); i++ ) {
+ Data dat = _variables.get(varList.get(i));
if( dat instanceof MatrixObject )
- ((MatrixObject)dat).enableCleanup(varsState.get(var));
+ ((MatrixObject)dat).enableCleanup(varsState[i]);
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/14c410ce/src/main/java/org/apache/sysml/runtime/instructions/cp/FunctionCallCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/FunctionCallCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/FunctionCallCPInstruction.java
index b901dfc..e785196 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/FunctionCallCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/FunctionCallCPInstruction.java
@@ -20,10 +20,8 @@
package org.apache.sysml.runtime.instructions.cp;
import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
import java.util.HashSet;
-import java.util.LinkedList;
+import java.util.Map.Entry;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.lops.Lop;
@@ -43,32 +41,30 @@ import org.apache.sysml.runtime.instructions.InstructionUtils;
public class FunctionCallCPInstruction extends CPInstruction {
private String _functionName;
private String _namespace;
+ private final CPOperand[] _boundInputs;
+ private final ArrayList<String> _boundInputNames;
+ private final ArrayList<String> _boundOutputNames;
+ private HashSet<String> _expectRetVars = null;
- public String getFunctionName() {
- return _functionName;
- }
-
- public String getNamespace() {
- return _namespace;
- }
-
- // stores both the bound input and output parameters
- private ArrayList<CPOperand> _boundInputParamOperands;
- private ArrayList<String> _boundInputParamNames;
- private ArrayList<String> _boundOutputParamNames;
-
- private FunctionCallCPInstruction(String namespace, String functName, ArrayList<CPOperand> boundInParamOperands,
- ArrayList<String> boundInParamNames, ArrayList<String> boundOutParamNames, String istr) {
+ private FunctionCallCPInstruction(String namespace, String functName, CPOperand[] boundInputs,
+ ArrayList<String> boundInputNames, ArrayList<String> boundOutputNames, String istr) {
super(null, functName, istr);
_cptype = CPINSTRUCTION_TYPE.External;
_functionName = functName;
_namespace = namespace;
- _boundInputParamOperands = boundInParamOperands;
- _boundInputParamNames = boundInParamNames;
- _boundOutputParamNames = boundOutParamNames;
+ _boundInputs = boundInputs;
+ _boundInputNames = boundInputNames;
+ _boundOutputNames = boundOutputNames;
+ }
+ public String getFunctionName() {
+ return _functionName;
}
+ public String getNamespace() {
+ return _namespace;
+ }
+
public static FunctionCallCPInstruction parseInstruction(String str)
throws DMLRuntimeException
{
@@ -78,20 +74,17 @@ public class FunctionCallCPInstruction extends CPInstruction {
String functionName = parts[2];
int numInputs = Integer.valueOf(parts[3]);
int numOutputs = Integer.valueOf(parts[4]);
- ArrayList<CPOperand> boundInParamOperands = new ArrayList<>();
- ArrayList<String> boundInParamNames = new ArrayList<>();
- ArrayList<String> boundOutParamNames = new ArrayList<>();
+ CPOperand[] boundInputs = new CPOperand[numInputs];
+ ArrayList<String> boundInputNames = new ArrayList<>();
+ ArrayList<String> boundOutputNames = new ArrayList<>();
for (int i = 0; i < numInputs; i++) {
- CPOperand operand = new CPOperand(parts[5 + i]);
- boundInParamOperands.add(operand);
- boundInParamNames.add(operand.getName());
+ boundInputs[i] = new CPOperand(parts[5 + i]);
+ boundInputNames.add(boundInputs[i].getName());
}
- for (int i = 0; i < numOutputs; i++) {
- boundOutParamNames.add(parts[5 + numInputs + i]);
- }
-
- return new FunctionCallCPInstruction ( namespace,functionName,
- boundInParamOperands, boundInParamNames, boundOutParamNames, str );
+ for (int i = 0; i < numOutputs; i++)
+ boundOutputNames.add(parts[5 + numInputs + i]);
+ return new FunctionCallCPInstruction ( namespace,
+ functionName, boundInputs, boundInputNames, boundOutputNames, str );
}
@Override
@@ -120,10 +113,10 @@ public class FunctionCallCPInstruction extends CPInstruction {
// get the function program block (stored in the Program object)
FunctionProgramBlock fpb = ec.getProgram().getFunctionProgramBlock(_namespace, _functionName);
- // sanity check number of function paramters
- if( _boundInputParamNames.size() < fpb.getInputParams().size() ) {
+ // sanity check number of function parameters
+ if( _boundInputs.length < fpb.getInputParams().size() ) {
throw new DMLRuntimeException("Number of bound input parameters does not match the function signature "
- + "("+_boundInputParamNames.size()+", but "+fpb.getInputParams().size()+" expected)");
+ + "("+_boundInputs.length+", but "+fpb.getInputParams().size()+" expected)");
}
// create bindings to formal parameters for given function call
@@ -131,35 +124,31 @@ public class FunctionCallCPInstruction extends CPInstruction {
LocalVariableMap functionVariables = new LocalVariableMap();
for( int i=0; i<fpb.getInputParams().size(); i++)
{
- DataIdentifier currFormalParam = fpb.getInputParams().get(i);
- String currFormalParamName = currFormalParam.getName();
- Data currFormalParamValue = null;
-
- CPOperand operand = _boundInputParamOperands.get(i);
- String varname = operand.getName();
//error handling non-existing variables
- if( !operand.isLiteral() && !ec.containsVariable(varname) ) {
- throw new DMLRuntimeException("Input variable '"+varname+"' not existing on call of " +
+ CPOperand input = _boundInputs[i];
+ if( !input.isLiteral() && !ec.containsVariable(input.getName()) ) {
+ throw new DMLRuntimeException("Input variable '"+input.getName()+"' not existing on call of " +
DMLProgram.constructFunctionKey(_namespace, _functionName) + " (line "+getLineNum()+").");
}
//get input matrix/frame/scalar
- currFormalParamValue = (operand.getDataType()!=DataType.SCALAR) ? ec.getVariable(varname) :
- ec.getScalarInput(varname, operand.getValueType(), operand.isLiteral());
+ DataIdentifier currFormalParam = fpb.getInputParams().get(i);
+ Data value = ec.getVariable(input);
//graceful value type conversion for scalar inputs with wrong type
- if( currFormalParamValue.getDataType() == DataType.SCALAR
- && currFormalParamValue.getValueType() != currFormalParam.getValueType() )
+ if( value.getDataType() == DataType.SCALAR
+ && value.getValueType() != currFormalParam.getValueType() )
{
- currFormalParamValue = ScalarObjectFactory.createScalarObject(
- currFormalParam.getValueType(), (ScalarObject) currFormalParamValue);
+ value = ScalarObjectFactory.createScalarObject(
+ currFormalParam.getValueType(), (ScalarObject)value);
}
- functionVariables.put(currFormalParamName, currFormalParamValue);
+ //set input parameter
+ functionVariables.put(currFormalParam.getName(), value);
}
// Pin the input variables so that they do not get deleted
// from pb's symbol table at the end of execution of function
- HashMap<String,Boolean> pinStatus = ec.pinVariables(_boundInputParamNames);
+ boolean[] pinStatus = ec.pinVariables(_boundInputNames);
// Create a symbol table under a new execution context for the function invocation,
// and copy the function arguments into the created table.
@@ -182,29 +171,29 @@ public class FunctionCallCPInstruction extends CPInstruction {
String fname = DMLProgram.constructFunctionKey(_namespace, _functionName);
throw new DMLRuntimeException("error executing function " + fname, e);
}
- LocalVariableMap retVars = fn_ec.getVariables();
// cleanup all returned variables w/o binding
- Collection<String> retVarnames = new LinkedList<>(retVars.keySet());
- HashSet<String> probeVars = new HashSet<>();
- for(DataIdentifier di : fpb.getOutputParams())
- probeVars.add(di.getName());
- for( String var : retVarnames ) {
- if( !probeVars.contains(var) ) //cleanup candidate
- {
- Data dat = fn_ec.removeVariable(var);
- if( dat != null && dat instanceof MatrixObject )
- fn_ec.cleanupMatrixObject((MatrixObject)dat);
- }
+ if( _expectRetVars == null ) {
+ _expectRetVars = new HashSet<>();
+ for(DataIdentifier di : fpb.getOutputParams())
+ _expectRetVars.add(di.getName());
+ }
+
+ LocalVariableMap retVars = fn_ec.getVariables();
+ for( Entry<String,Data> var : retVars.entrySet() ) {
+ if( _expectRetVars.contains(var.getKey()) )
+ continue;
+ //cleanup unexpected return values to avoid leaks
+ if( var.getValue() instanceof MatrixObject )
+ fn_ec.cleanupMatrixObject((MatrixObject)var.getValue());
}
// Unpin the pinned variables
- ec.unpinVariables(_boundInputParamNames, pinStatus);
+ ec.unpinVariables(_boundInputNames, pinStatus);
// add the updated binding for each return variable to the variables in original symbol table
for (int i=0; i< fpb.getOutputParams().size(); i++){
-
- String boundVarName = _boundOutputParamNames.get(i);
+ String boundVarName = _boundOutputNames.get(i);
Data boundValue = retVars.get(fpb.getOutputParams().get(i).getName());
if (boundValue == null)
throw new DMLRuntimeException(boundVarName + " was not assigned a return value");
@@ -240,14 +229,12 @@ public class FunctionCallCPInstruction extends CPInstruction {
LOG.debug("ExternalBuiltInFunction: " + this.toString());
}
- public ArrayList<String> getBoundInputParamNames()
- {
- return _boundInputParamNames;
+ public ArrayList<String> getBoundInputParamNames() {
+ return _boundInputNames;
}
- public ArrayList<String> getBoundOutputParamNames()
- {
- return _boundOutputParamNames;
+ public ArrayList<String> getBoundOutputParamNames() {
+ return _boundOutputNames;
}
public void setFunctionName(String fname)
@@ -277,6 +264,4 @@ public class FunctionCallCPInstruction extends CPInstruction {
return sb.substring( 0, sb.length()-Lop.OPERAND_DELIMITOR.length() );
}
-
-
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/14c410ce/src/test/java/org/apache/sysml/test/integration/functions/compress/CompressedLinregCG.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/compress/CompressedLinregCG.java b/src/test/java/org/apache/sysml/test/integration/functions/compress/CompressedLinregCG.java
index 6e2ddef..a7e0971 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/compress/CompressedLinregCG.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/compress/CompressedLinregCG.java
@@ -100,24 +100,23 @@ public class CompressedLinregCG extends AutomatedTestBase
try
{
- String TEST_NAME = testname;
- TestConfiguration config = getTestConfiguration(TEST_NAME);
+ TestConfiguration config = getTestConfiguration(testname);
+ loadTestConfiguration(config);
/* This is for running the junit test the new way, i.e., construct the arguments directly */
- String HOME = SCRIPT_DIR + "functions/codegen/";
+ String HOME1 = SCRIPT_DIR + "functions/compress/";
+ String HOME2 = SCRIPT_DIR + "functions/codegen/";
fullDMLScriptName = "scripts/algorithms/LinearRegCG.dml";
programArgs = new String[]{ "-explain", "-stats", "-nvargs", "X="+input("X"), "Y="+input("y"),
"icpt="+String.valueOf(intercept), "tol="+String.valueOf(epsilon),
"maxi="+String.valueOf(maxiter), "reg="+String.valueOf(regular), "B="+output("w")};
- fullRScriptName = HOME + "Algorithm_LinregCG.R";
+ fullRScriptName = HOME2 + "Algorithm_LinregCG.R";
rCmd = "Rscript" + " " + fullRScriptName + " " +
- HOME + INPUT_DIR + " " +
- String.valueOf(intercept) + " " + String.valueOf(epsilon) + " " +
- String.valueOf(maxiter) + " " + String.valueOf(regular) + HOME + EXPECTED_DIR;
+ HOME1 + INPUT_DIR + " " +
+ String.valueOf(intercept) + " " + String.valueOf(epsilon) + " " +
+ String.valueOf(maxiter) + " " + String.valueOf(regular) + " "+ HOME1 + EXPECTED_DIR;
- loadTestConfiguration(config);
-
//generate actual datasets
double[][] X = getRandomMatrix(rows, cols, 1, 1, sparse?sparsity2:sparsity1, 7);
writeInputMatrixWithMTD("X", X, true);
@@ -141,7 +140,7 @@ public class CompressedLinregCG extends AutomatedTestBase
finally {
rtplatform = platformOld;
DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
- InfrastructureAnalyzer.setLocalMaxMemory(memOld);
+ InfrastructureAnalyzer.setLocalMaxMemory(memOld);
CompressedMatrixBlock.ALLOW_DDC_ENCODING = true;
}
}
[2/5] systemml git commit: [MINOR] Performance createvar,
rmvar and cleanup meta data handling
Posted by mb...@apache.org.
[MINOR] Performance createvar, rmvar and cleanup meta data handling
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/d7d312c3
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/d7d312c3
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/d7d312c3
Branch: refs/heads/master
Commit: d7d312c39b5c6c90ca6bc487e6198eabf229dfc1
Parents: f76f213
Author: Matthias Boehm <mb...@gmail.com>
Authored: Thu Nov 2 22:59:23 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Fri Nov 3 18:59:28 2017 -0700
----------------------------------------------------------------------
.../apache/sysml/api/jmlc/PreparedScript.java | 6 +-
.../apache/sysml/api/mlcontext/MLContext.java | 6 +-
.../api/mlcontext/MLContextConversionUtil.java | 20 ++---
.../apache/sysml/hops/cost/CostEstimator.java | 3 +-
.../sysml/hops/ipa/InterProceduralAnalysis.java | 6 +-
.../apache/sysml/hops/recompile/Recompiler.java | 10 +--
.../RewriteRemovePersistentReadWrite.java | 6 +-
.../apache/sysml/lops/runtime/RunMRJobs.java | 7 +-
.../ExternalFunctionProgramBlock.java | 4 +-
.../ExternalFunctionProgramBlockCP.java | 4 +-
.../controlprogram/caching/CacheableData.java | 20 +++--
.../controlprogram/caching/FrameObject.java | 11 ++-
.../controlprogram/caching/MatrixObject.java | 58 +++++++--------
.../context/ExecutionContext.java | 46 +++++-------
.../controlprogram/parfor/DataPartitioner.java | 6 +-
.../controlprogram/parfor/ProgramConverter.java | 6 +-
.../parfor/ResultMergeLocalFile.java | 22 +++---
.../parfor/ResultMergeLocalMemory.java | 6 +-
.../parfor/ResultMergeRemoteMR.java | 6 +-
.../parfor/ResultMergeRemoteSpark.java | 6 +-
.../parfor/opt/OptimizerRuleBased.java | 4 +-
.../runtime/instructions/MRJobInstruction.java | 15 ++--
.../cp/DataPartitionCPInstruction.java | 4 +-
.../cp/QuantilePickCPInstruction.java | 12 +--
.../instructions/cp/VariableCPInstruction.java | 21 ++----
.../cpfile/MatrixIndexingCPFileInstruction.java | 6 +-
.../ParameterizedBuiltinCPFileInstruction.java | 8 +-
.../spark/CSVReblockSPInstruction.java | 4 +-
.../spark/ConvolutionSPInstruction.java | 4 +-
.../spark/ReblockSPInstruction.java | 4 +-
.../org/apache/sysml/runtime/matrix/GMR.java | 9 +--
.../apache/sysml/runtime/matrix/JobReturn.java | 27 ++-----
.../runtime/matrix/MatrixCharacteristics.java | 27 +++----
.../matrix/MatrixDimensionsMetaData.java | 78 --------------------
.../runtime/matrix/MatrixFormatMetaData.java | 53 -------------
.../apache/sysml/runtime/matrix/MetaData.java | 40 ++++++----
.../sysml/runtime/matrix/MetaDataFormat.java | 49 ++++++++++++
.../matrix/MetaDataNumItemsByEachReducer.java | 53 +++++++++++++
.../data/NumItemsByEachReducerMetaData.java | 64 ----------------
.../matrix/sort/PickFromCompactInputFormat.java | 8 +-
.../sysml/runtime/util/MapReduceTool.java | 8 +-
.../sysml/runtime/util/UtilFunctions.java | 6 +-
src/main/java/org/apache/sysml/udf/Matrix.java | 4 +-
.../sysml/yarn/ropt/ResourceOptimizer.java | 5 +-
.../functions/frame/FrameEvictionTest.java | 4 +-
45 files changed, 327 insertions(+), 449 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java b/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
index f14d3a1..c712bb2 100644
--- a/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
+++ b/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
@@ -49,7 +49,7 @@ import org.apache.sysml.runtime.instructions.cp.IntObject;
import org.apache.sysml.runtime.instructions.cp.ScalarObject;
import org.apache.sysml.runtime.instructions.cp.StringObject;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
@@ -258,7 +258,7 @@ public class PreparedScript
//create new matrix object
MatrixCharacteristics mc = new MatrixCharacteristics(matrix.getNumRows(), matrix.getNumColumns(), blocksize, blocksize);
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
+ MetaDataFormat meta = new MetaDataFormat(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
MatrixObject mo = new MatrixObject(ValueType.DOUBLE, OptimizerUtils.getUniqueTempFileName(), meta);
mo.acquireModify(matrix);
mo.release();
@@ -365,7 +365,7 @@ public class PreparedScript
//create new frame object
MatrixCharacteristics mc = new MatrixCharacteristics(frame.getNumRows(), frame.getNumColumns(), -1, -1);
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc, OutputInfo.BinaryCellOutputInfo, InputInfo.BinaryCellInputInfo);
+ MetaDataFormat meta = new MetaDataFormat(mc, OutputInfo.BinaryCellOutputInfo, InputInfo.BinaryCellInputInfo);
FrameObject fo = new FrameObject(OptimizerUtils.getUniqueTempFileName(), meta);
fo.acquireModify(frame);
fo.release();
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java b/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
index 6fe66ff..37bd309 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
@@ -39,7 +39,7 @@ import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
import org.apache.sysml.runtime.instructions.cp.Data;
import org.apache.sysml.runtime.instructions.cp.ScalarObject;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
import org.apache.sysml.utils.Explain.ExplainType;
import org.apache.sysml.utils.MLContextProxy;
@@ -537,8 +537,8 @@ public class MLContext {
exp.addVarParam(DataExpression.DATATYPEPARAM, new StringIdentifier("matrix", source));
exp.addVarParam(DataExpression.VALUETYPEPARAM, new StringIdentifier("double", source));
- if (mo.getMetaData() instanceof MatrixFormatMetaData) {
- MatrixFormatMetaData metaData = (MatrixFormatMetaData) mo.getMetaData();
+ if (mo.getMetaData() instanceof MetaDataFormat) {
+ MetaDataFormat metaData = (MetaDataFormat) mo.getMetaData();
if (metaData.getOutputInfo() == OutputInfo.CSVOutputInfo) {
exp.addVarParam(DataExpression.FORMAT_TYPE,
new StringIdentifier(DataExpression.FORMAT_TYPE_VALUE_CSV, source));
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java b/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java
index 6112a8a..41e8551 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java
@@ -56,7 +56,7 @@ import org.apache.sysml.runtime.instructions.spark.utils.FrameRDDConverterUtils;
import org.apache.sysml.runtime.instructions.spark.utils.RDDConverterUtils;
import org.apache.sysml.runtime.instructions.spark.utils.SparkUtils;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.IJV;
import org.apache.sysml.runtime.matrix.data.InputInfo;
@@ -111,7 +111,7 @@ public class MLContextConversionUtil {
ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
MatrixObject matrixObject = new MatrixObject(ValueType.DOUBLE, OptimizerUtils.getUniqueTempFileName(),
- new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo));
+ new MetaDataFormat(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo));
matrixObject.acquireModify(matrixBlock);
matrixObject.release();
@@ -165,7 +165,7 @@ public class MLContextConversionUtil {
MatrixCharacteristics mc = (matrixMetadata != null) ? matrixMetadata.asMatrixCharacteristics()
: new MatrixCharacteristics();
MatrixObject matrixObject = new MatrixObject(ValueType.DOUBLE, OptimizerUtils.getUniqueTempFileName(),
- new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo));
+ new MetaDataFormat(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo));
matrixObject.acquireModify(matrixBlock);
matrixObject.release();
return matrixObject;
@@ -190,7 +190,7 @@ public class MLContextConversionUtil {
try {
MatrixCharacteristics mc = (frameMetadata != null) ? frameMetadata.asMatrixCharacteristics()
: new MatrixCharacteristics();
- MatrixFormatMetaData mtd = new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo,
+ MetaDataFormat mtd = new MetaDataFormat(mc, OutputInfo.BinaryBlockOutputInfo,
InputInfo.BinaryBlockInputInfo);
FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(), mtd,
frameMetadata.getFrameSchema().getSchema().toArray(new ValueType[0]));
@@ -246,7 +246,7 @@ public class MLContextConversionUtil {
JavaPairRDD<MatrixIndexes, MatrixBlock> javaPairRdd = SparkUtils.copyBinaryBlockMatrix(binaryBlocks, copy);
MatrixObject matrixObject = new MatrixObject(ValueType.DOUBLE, OptimizerUtils.getUniqueTempFileName(),
- new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo));
+ new MetaDataFormat(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo));
matrixObject.setRDDHandle(new RDDObject(javaPairRdd, variableName));
return matrixObject;
}
@@ -315,7 +315,7 @@ public class MLContextConversionUtil {
UtilFunctions.nCopies((int)mc.getCols(), ValueType.STRING);
FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(),
- new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo), schema);
+ new MetaDataFormat(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo), schema);
frameObject.setRDDHandle(new RDDObject(binaryBlocks, variableName));
return frameObject;
}
@@ -618,7 +618,7 @@ public class MLContextConversionUtil {
: new MatrixCharacteristics();
MatrixObject matrixObject = new MatrixObject(ValueType.DOUBLE, OptimizerUtils.getUniqueTempFileName(),
- new MatrixFormatMetaData(mc, OutputInfo.CSVOutputInfo, InputInfo.CSVInputInfo));
+ new MetaDataFormat(mc, OutputInfo.CSVOutputInfo, InputInfo.CSVInputInfo));
JavaPairRDD<LongWritable, Text> javaPairRDD2 = javaPairRDD.mapToPair(new CopyTextInputFunction());
matrixObject.setRDDHandle(new RDDObject(javaPairRDD2, variableName));
return matrixObject;
@@ -656,7 +656,7 @@ public class MLContextConversionUtil {
JavaPairRDD<LongWritable, Text> javaPairRDDText = javaPairRDD.mapToPair(new CopyTextInputFunction());
FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(),
- new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo),
+ new MetaDataFormat(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo),
frameMetadata.getFrameSchema().getSchema().toArray(new ValueType[0]));
JavaPairRDD<Long, FrameBlock> rdd;
try {
@@ -689,7 +689,7 @@ public class MLContextConversionUtil {
: new MatrixCharacteristics();
MatrixObject matrixObject = new MatrixObject(ValueType.DOUBLE, OptimizerUtils.getUniqueTempFileName(),
- new MatrixFormatMetaData(mc, OutputInfo.TextCellOutputInfo, InputInfo.TextCellInputInfo));
+ new MetaDataFormat(mc, OutputInfo.TextCellOutputInfo, InputInfo.TextCellInputInfo));
JavaPairRDD<LongWritable, Text> javaPairRDD2 = javaPairRDD.mapToPair(new CopyTextInputFunction());
matrixObject.setRDDHandle(new RDDObject(javaPairRDD2, variableName));
return matrixObject;
@@ -716,7 +716,7 @@ public class MLContextConversionUtil {
JavaPairRDD<LongWritable, Text> javaPairRDDText = javaPairRDD.mapToPair(new CopyTextInputFunction());
FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(),
- new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo),
+ new MetaDataFormat(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo),
frameMetadata.getFrameSchema().getSchema().toArray(new ValueType[0]));
JavaPairRDD<Long, FrameBlock> rdd;
try {
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java b/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
index ea1e3d6..7b5f20e 100644
--- a/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
+++ b/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
@@ -60,7 +60,6 @@ import org.apache.sysml.runtime.instructions.cp.UnaryCPInstruction;
import org.apache.sysml.runtime.instructions.cp.VariableCPInstruction;
import org.apache.sysml.runtime.instructions.mr.MRInstruction;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixDimensionsMetaData;
import org.apache.sysml.runtime.matrix.operators.CMOperator;
import org.apache.sysml.runtime.matrix.operators.CMOperator.AggregateOperationTypes;
import org.apache.sysml.runtime.util.UtilFunctions;
@@ -227,7 +226,7 @@ public abstract class CostEstimator
if( dat instanceof MatrixObject ) //matrix
{
MatrixObject mo = (MatrixObject) dat;
- MatrixCharacteristics mc = ((MatrixDimensionsMetaData)mo.getMetaData()).getMatrixCharacteristics();
+ MatrixCharacteristics mc = mo.getMatrixCharacteristics();
long rlen = mc.getRows();
long clen = mc.getCols();
long brlen = mc.getRowsPerBlock();
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java b/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
index 2ab6b3c..de12bb1 100644
--- a/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
+++ b/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
@@ -58,7 +58,7 @@ import org.apache.sysml.runtime.instructions.cp.Data;
import org.apache.sysml.runtime.instructions.cp.ScalarObject;
import org.apache.sysml.runtime.instructions.cp.ScalarObjectFactory;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.udf.lib.DynamicReadMatrixCP;
import org.apache.sysml.udf.lib.DynamicReadMatrixRcCP;
import org.apache.sysml.udf.lib.OrderWrapper;
@@ -554,7 +554,7 @@ public class InterProceduralAnalysis
MatrixCharacteristics mc = new MatrixCharacteristics( input.getDim1(), input.getDim2(),
ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(),
fcallSizes.isSafeNnz(fkey, i)?input.getNnz():-1 );
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,null,null);
+ MetaDataFormat meta = new MetaDataFormat(mc,null,null);
mo.setMetaData(meta);
vars.put(dat.getName(), mo);
}
@@ -729,7 +729,7 @@ public class InterProceduralAnalysis
MatrixObject moOut = new MatrixObject(ValueType.DOUBLE, null);
MatrixCharacteristics mc = new MatrixCharacteristics( dim1, dim2,
ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), nnz);
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,null,null);
+ MetaDataFormat meta = new MetaDataFormat(mc,null,null);
moOut.setMetaData(meta);
return moOut;
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
index 3c1f6c9..c6d74dd 100644
--- a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
+++ b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
@@ -96,7 +96,7 @@ import org.apache.sysml.runtime.instructions.mr.RandInstruction;
import org.apache.sysml.runtime.instructions.mr.SeqInstruction;
import org.apache.sysml.runtime.io.IOUtilFunctions;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
@@ -1120,7 +1120,7 @@ public class Recompiler
int blksz = ConfigurationManager.getBlocksize();
MatrixCharacteristics mc = new MatrixCharacteristics(
dim1, dim2, blksz, blksz, nnz);
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,null,null);
+ MetaDataFormat meta = new MetaDataFormat(mc,null,null);
moOut.setMetaData(meta);
return moOut;
@@ -1357,7 +1357,7 @@ public class Recompiler
hop.getDim1(), hop.getDim2(),
ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(),
hop.getNnz());
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,null,null);
+ MetaDataFormat meta = new MetaDataFormat(mc,null,null);
mo.setMetaData(meta);
vars.put(varName, mo);
}
@@ -1751,7 +1751,7 @@ public class Recompiler
if( ret && !localMode ) {
for( MatrixObject mo : inputs )
{
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) mo.getMetaData();
+ MetaDataFormat iimd = (MetaDataFormat) mo.getMetaData();
if(( iimd.getInputInfo()==InputInfo.TextCellInputInfo
|| iimd.getInputInfo()==InputInfo.MatrixMarketInputInfo
|| iimd.getInputInfo()==InputInfo.CSVInputInfo
@@ -1805,7 +1805,7 @@ public class Recompiler
//robustness for usage through mlcontext (key/values of input rdds are
//not serializable for text; also bufferpool rdd read only supported for
// binarycell and binaryblock)
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) obj.getMetaData();
+ MetaDataFormat iimd = (MetaDataFormat) obj.getMetaData();
if( obj.getRDDHandle() != null
&& iimd.getInputInfo() != InputInfo.BinaryBlockInputInfo
&& iimd.getInputInfo() != InputInfo.BinaryCellInputInfo ) {
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemovePersistentReadWrite.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemovePersistentReadWrite.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemovePersistentReadWrite.java
index 34743bb..ec5c6c0 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemovePersistentReadWrite.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemovePersistentReadWrite.java
@@ -34,7 +34,7 @@ import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
import org.apache.sysml.runtime.controlprogram.caching.CacheableData;
import org.apache.sysml.runtime.instructions.cp.Data;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.MetaData;
import org.apache.sysml.runtime.matrix.data.InputInfo;
@@ -131,8 +131,8 @@ public class RewriteRemovePersistentReadWrite extends HopRewriteRule
//disable unnecessary reblock of binary block w/ equal block sizes
if( dop.requiresReblock() && _inputsMeta.containsKey(dop.getName())
- && _inputsMeta.get(dop.getName()) instanceof MatrixFormatMetaData) {
- MatrixFormatMetaData meta = (MatrixFormatMetaData)_inputsMeta.get(dop.getName());
+ && _inputsMeta.get(dop.getName()) instanceof MetaDataFormat) {
+ MetaDataFormat meta = (MetaDataFormat)_inputsMeta.get(dop.getName());
MatrixCharacteristics mc = meta.getMatrixCharacteristics();
boolean matchingBlksz = mc.getRowsPerBlock() == dop.getRowsInBlock()
&& mc.getColsPerBlock() == dop.getColsInBlock();
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/lops/runtime/RunMRJobs.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/runtime/RunMRJobs.java b/src/main/java/org/apache/sysml/lops/runtime/RunMRJobs.java
index 5cd0f60..2c04425 100644
--- a/src/main/java/org/apache/sysml/lops/runtime/RunMRJobs.java
+++ b/src/main/java/org/apache/sysml/lops/runtime/RunMRJobs.java
@@ -59,8 +59,7 @@ import org.apache.sysml.runtime.matrix.JobReturn;
import org.apache.sysml.runtime.matrix.MMCJMR;
import org.apache.sysml.runtime.matrix.MMRJMR;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixDimensionsMetaData;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.ReblockMR;
import org.apache.sysml.runtime.matrix.SortMR;
import org.apache.sysml.runtime.matrix.WriteCSVMR;
@@ -324,7 +323,7 @@ public class RunMRJobs
{
for (int i = 0; i < outputMatrices.length; i++) {
//get output meta data
- MatrixFormatMetaData meta = (MatrixFormatMetaData)outputMatrices[i].getMetaData();
+ MetaDataFormat meta = (MetaDataFormat)outputMatrices[i].getMetaData();
MatrixCharacteristics mc = meta.getMatrixCharacteristics();
OutputInfo outinfo = meta.getOutputInfo();
String fname = outputMatrices[i].getFileName();
@@ -345,7 +344,7 @@ public class RunMRJobs
// Currently, valueType information in not stored in MR instruction,
// since only DOUBLE matrices are supported ==> hard coded the value type information for now
MapReduceTool.writeMetaDataFile(fname + ".mtd", ValueType.DOUBLE,
- ((MatrixDimensionsMetaData)ret.getMetaData(i)).getMatrixCharacteristics(), outinfo);
+ ret.getMetaData(i).getMatrixCharacteristics(), outinfo);
}
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
index d84abfa..9738203 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
@@ -49,7 +49,7 @@ import org.apache.sysml.runtime.instructions.cp.ScalarObject;
import org.apache.sysml.runtime.instructions.cp.StringObject;
import org.apache.sysml.runtime.instructions.cp.VariableCPInstruction;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
import org.apache.sysml.udf.ExternalFunctionInvocationInstruction;
@@ -717,7 +717,7 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
throws CacheException
{
MatrixCharacteristics mc = new MatrixCharacteristics(m.getNumRows(),m.getNumCols(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
- MatrixFormatMetaData mfmd = new MatrixFormatMetaData(mc, OutputInfo.TextCellOutputInfo, InputInfo.TextCellInputInfo);
+ MetaDataFormat mfmd = new MetaDataFormat(mc, OutputInfo.TextCellOutputInfo, InputInfo.TextCellInputInfo);
return new MatrixObject(ValueType.DOUBLE, m.getFilePath(), mfmd);
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java
index 29806c2..8be1455 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java
@@ -31,7 +31,7 @@ import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
import org.apache.sysml.udf.ExternalFunctionInvocationInstruction;
@@ -168,7 +168,7 @@ public class ExternalFunctionProgramBlockCP extends ExternalFunctionProgramBlock
if( ret == null ) //otherwise, pass in-memory matrix from extfunct back to invoking program
{
MatrixCharacteristics mc = new MatrixCharacteristics(m.getNumRows(),m.getNumCols(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
- MatrixFormatMetaData mfmd = new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
+ MetaDataFormat mfmd = new MetaDataFormat(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
ret = new MatrixObject(ValueType.DOUBLE, m.getFilePath(), mfmd);
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
index c7ffdb1..e41b06c 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
@@ -47,13 +47,12 @@ import org.apache.sysml.runtime.instructions.spark.data.BroadcastObject;
import org.apache.sysml.runtime.instructions.spark.data.RDDObject;
import org.apache.sysml.runtime.io.IOUtilFunctions;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixDimensionsMetaData;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
+import org.apache.sysml.runtime.matrix.MetaDataNumItemsByEachReducer;
import org.apache.sysml.runtime.matrix.MetaData;
import org.apache.sysml.runtime.matrix.data.FileFormatProperties;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
-import org.apache.sysml.runtime.matrix.data.NumItemsByEachReducerMetaData;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
import org.apache.sysml.runtime.util.LocalFileUtils;
import org.apache.sysml.runtime.util.MapReduceTool;
@@ -314,8 +313,7 @@ public abstract class CacheableData<T extends CacheBlock> extends Data
}
public MatrixCharacteristics getMatrixCharacteristics() {
- MatrixDimensionsMetaData meta = (MatrixDimensionsMetaData) _metaData;
- return meta.getMatrixCharacteristics();
+ return _metaData.getMatrixCharacteristics();
}
public abstract void refreshMetaData()
@@ -990,7 +988,7 @@ public abstract class CacheableData<T extends CacheBlock> extends Data
protected T readBlobFromHDFS(String fname)
throws IOException
{
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) _metaData;
+ MetaDataFormat iimd = (MetaDataFormat) _metaData;
MatrixCharacteristics mc = iimd.getMatrixCharacteristics();
return readBlobFromHDFS(fname, mc.getRows(), mc.getCols());
}
@@ -1010,7 +1008,7 @@ public abstract class CacheableData<T extends CacheBlock> extends Data
protected void writeMetaData (String filePathAndName, String outputFormat, FileFormatProperties formatProperties)
throws DMLRuntimeException, IOException
{
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) _metaData;
+ MetaDataFormat iimd = (MetaDataFormat) _metaData;
if (iimd == null)
throw new DMLRuntimeException("Unexpected error while writing mtd file (" + filePathAndName + ") -- metadata is null.");
@@ -1042,7 +1040,7 @@ public abstract class CacheableData<T extends CacheBlock> extends Data
boolean ret = true;
if( outputFormat != null ) {
try {
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) _metaData;
+ MetaDataFormat iimd = (MetaDataFormat) _metaData;
OutputInfo oi1 = InputInfo.getMatchingOutputInfo( iimd.getInputInfo() );
OutputInfo oi2 = OutputInfo.stringToOutputInfo( outputFormat );
if( oi1 != oi2 )
@@ -1427,13 +1425,13 @@ public abstract class CacheableData<T extends CacheBlock> extends Data
str.append(": ");
str.append(_hdfsFileName + ", ");
- if (_metaData instanceof NumItemsByEachReducerMetaData) {
+ if (_metaData instanceof MetaDataNumItemsByEachReducer) {
str.append("NumItemsByEachReducerMetaData");
} else {
try {
- MatrixFormatMetaData md = (MatrixFormatMetaData) _metaData;
+ MetaDataFormat md = (MetaDataFormat) _metaData;
if (md != null) {
- MatrixCharacteristics mc = ((MatrixDimensionsMetaData) _metaData).getMatrixCharacteristics();
+ MatrixCharacteristics mc = _metaData.getMatrixCharacteristics();
str.append(mc.toString());
InputInfo ii = md.getInputInfo();
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
index 4cf3d7f..b594777 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
@@ -36,8 +36,7 @@ import org.apache.sysml.runtime.io.FrameReaderFactory;
import org.apache.sysml.runtime.io.FrameWriter;
import org.apache.sysml.runtime.io.FrameWriterFactory;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixDimensionsMetaData;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.MetaData;
import org.apache.sysml.runtime.matrix.data.FileFormatProperties;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
@@ -140,7 +139,7 @@ public class FrameObject extends CacheableData<FrameBlock>
throw new CacheException("Cannot refresh meta data because there is no data or meta data. ");
//update matrix characteristics
- MatrixCharacteristics mc = ((MatrixDimensionsMetaData) _metaData).getMatrixCharacteristics();
+ MatrixCharacteristics mc = _metaData.getMatrixCharacteristics();
mc.setDimension( _data.getNumRows(),_data.getNumColumns() );
mc.setNonZeros(_data.getNumRows()*_data.getNumColumns());
@@ -167,7 +166,7 @@ public class FrameObject extends CacheableData<FrameBlock>
protected FrameBlock readBlobFromHDFS(String fname, long rlen, long clen)
throws IOException
{
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) _metaData;
+ MetaDataFormat iimd = (MetaDataFormat) _metaData;
MatrixCharacteristics mc = iimd.getMatrixCharacteristics();
//handle missing schema if necessary
@@ -202,7 +201,7 @@ public class FrameObject extends CacheableData<FrameBlock>
//prepare return status (by default only collect)
status.setValue(false);
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) _metaData;
+ MetaDataFormat iimd = (MetaDataFormat) _metaData;
MatrixCharacteristics mc = iimd.getMatrixCharacteristics();
int rlen = (int)mc.getRows();
int clen = (int)mc.getCols();
@@ -247,7 +246,7 @@ public class FrameObject extends CacheableData<FrameBlock>
throws IOException, DMLRuntimeException
{
//prepare output info
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) _metaData;
+ MetaDataFormat iimd = (MetaDataFormat) _metaData;
OutputInfo oinfo = (ofmt != null ? OutputInfo.stringToOutputInfo (ofmt )
: InputInfo.getMatchingOutputInfo (iimd.getInputInfo ()));
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
index e9017e5..6486b02 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
@@ -35,8 +35,7 @@ import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartition
import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
import org.apache.sysml.runtime.instructions.spark.data.RDDObject;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixDimensionsMetaData;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.MetaData;
import org.apache.sysml.runtime.matrix.data.FileFormatProperties;
import org.apache.sysml.runtime.matrix.data.InputInfo;
@@ -115,8 +114,8 @@ public class MatrixObject extends CacheableData<MatrixBlock>
//base copy constructor
super(mo);
- MatrixFormatMetaData metaOld = (MatrixFormatMetaData)mo.getMetaData();
- _metaData = new MatrixFormatMetaData(new MatrixCharacteristics(metaOld.getMatrixCharacteristics()),
+ MetaDataFormat metaOld = (MetaDataFormat)mo.getMetaData();
+ _metaData = new MetaDataFormat(new MatrixCharacteristics(metaOld.getMatrixCharacteristics()),
metaOld.getOutputInfo(), metaOld.getInputInfo());
_updateType = mo._updateType;
@@ -136,7 +135,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
@Override
public void updateMatrixCharacteristics (MatrixCharacteristics mc) {
- ((MatrixDimensionsMetaData)_metaData).setMatrixCharacteristics( mc );
+ _metaData.getMatrixCharacteristics().set(mc);
}
/**
@@ -150,42 +149,35 @@ public class MatrixObject extends CacheableData<MatrixBlock>
{
if ( _data == null || _metaData ==null ) //refresh only for existing data
throw new CacheException("Cannot refresh meta data because there is no data or meta data. ");
- //we need to throw an exception, otherwise input/output format cannot be inferred
+ //we need to throw an exception, otherwise input/output format cannot be inferred
- MatrixCharacteristics mc = ((MatrixDimensionsMetaData) _metaData).getMatrixCharacteristics();
- mc.setDimension( _data.getNumRows(),
- _data.getNumColumns() );
- mc.setNonZeros( _data.getNonZeros() );
+ MatrixCharacteristics mc = _metaData.getMatrixCharacteristics();
+ mc.setDimension( _data.getNumRows(), _data.getNumColumns() );
+ mc.setNonZeros( _data.getNonZeros() );
}
public long getNumRows() {
- MatrixCharacteristics mc = getMatrixCharacteristics();
- return mc.getRows ();
+ return getMatrixCharacteristics().getRows();
}
public long getNumColumns() {
- MatrixCharacteristics mc = getMatrixCharacteristics();
- return mc.getCols ();
+ return getMatrixCharacteristics().getCols();
}
public long getNumRowsPerBlock() {
- MatrixCharacteristics mc = getMatrixCharacteristics();
- return mc.getRowsPerBlock();
+ return getMatrixCharacteristics().getRowsPerBlock();
}
public long getNumColumnsPerBlock() {
- MatrixCharacteristics mc = getMatrixCharacteristics();
- return mc.getColsPerBlock();
+ return getMatrixCharacteristics().getColsPerBlock();
}
public long getNnz() {
- MatrixCharacteristics mc = getMatrixCharacteristics();
- return mc.getNonZeros();
+ return getMatrixCharacteristics().getNonZeros();
}
public double getSparsity() {
- MatrixCharacteristics mc = getMatrixCharacteristics();
- return OptimizerUtils.getSparsity(mc);
+ return OptimizerUtils.getSparsity(getMatrixCharacteristics());
}
// *********************************************
@@ -264,7 +256,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
boolean blockwise = (_partitionFormat==PDataPartitionFormat.ROW_BLOCK_WISE || _partitionFormat==PDataPartitionFormat.COLUMN_BLOCK_WISE);
//preparations for block wise access
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) _metaData;
+ MetaDataFormat iimd = (MetaDataFormat) _metaData;
MatrixCharacteristics mc = iimd.getMatrixCharacteristics();
int brlen = mc.getRowsPerBlock();
int bclen = mc.getColsPerBlock();
@@ -427,7 +419,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
protected MatrixBlock readBlobFromHDFS(String fname, long rlen, long clen)
throws IOException
{
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) _metaData;
+ MetaDataFormat iimd = (MetaDataFormat) _metaData;
MatrixCharacteristics mc = iimd.getMatrixCharacteristics();
long begin = 0;
@@ -464,7 +456,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
//prepare return status (by default only collect)
writeStatus.setValue(false);
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) _metaData;
+ MetaDataFormat iimd = (MetaDataFormat) _metaData;
MatrixCharacteristics mc = iimd.getMatrixCharacteristics();
InputInfo ii = iimd.getInputInfo();
MatrixBlock mb = null;
@@ -489,7 +481,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
//note: lazy, partition-at-a-time collect (toLocalIterator) was significantly slower
if( !MapReduceTool.existsFileOnHDFS(_hdfsFileName) ) { //prevent overwrite existing file
long newnnz = SparkExecutionContext.writeRDDtoHDFS(lrdd, _hdfsFileName, iimd.getOutputInfo());
- ((MatrixDimensionsMetaData) _metaData).getMatrixCharacteristics().setNonZeros(newnnz);
+ _metaData.getMatrixCharacteristics().setNonZeros(newnnz);
((RDDObject)rdd).setPending(false); //mark rdd as non-pending (for export)
((RDDObject)rdd).setHDFSFile(true); //mark rdd as hdfs file (for restore)
writeStatus.setValue(true); //mark for no cache-write on read
@@ -533,7 +525,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
begin = System.currentTimeMillis();
}
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) _metaData;
+ MetaDataFormat iimd = (MetaDataFormat) _metaData;
if (_data != null)
{
@@ -569,14 +561,14 @@ public class MatrixObject extends CacheableData<MatrixBlock>
protected void writeBlobFromRDDtoHDFS(RDDObject rdd, String fname, String outputFormat)
throws IOException, DMLRuntimeException
{
- //prepare output info
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) _metaData;
- OutputInfo oinfo = (outputFormat != null ? OutputInfo.stringToOutputInfo (outputFormat)
- : InputInfo.getMatchingOutputInfo (iimd.getInputInfo ()));
-
+ //prepare output info
+ MetaDataFormat iimd = (MetaDataFormat) _metaData;
+ OutputInfo oinfo = (outputFormat != null ? OutputInfo.stringToOutputInfo (outputFormat) :
+ InputInfo.getMatchingOutputInfo (iimd.getInputInfo ()));
+
//note: the write of an RDD to HDFS might trigger
//lazy evaluation of pending transformations.
long newnnz = SparkExecutionContext.writeRDDtoHDFS(rdd, fname, oinfo);
- ((MatrixDimensionsMetaData) _metaData).getMatrixCharacteristics().setNonZeros(newnnz);
+ _metaData.getMatrixCharacteristics().setNonZeros(newnnz);
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
index 4cd371b..79a658d 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
@@ -48,8 +48,7 @@ import org.apache.sysml.runtime.instructions.cp.ScalarObjectFactory;
import org.apache.sysml.runtime.instructions.gpu.context.GPUContext;
import org.apache.sysml.runtime.instructions.gpu.context.GPUObject;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixDimensionsMetaData;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.MetaData;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
@@ -233,8 +232,7 @@ public class ExecutionContext {
public MatrixCharacteristics getMatrixCharacteristics( String varname )
throws DMLRuntimeException
{
- MatrixDimensionsMetaData dims = (MatrixDimensionsMetaData) getMetaData(varname);
- return dims.getMatrixCharacteristics();
+ return getMetaData(varname).getMatrixCharacteristics();
}
/**
@@ -280,14 +278,14 @@ public class ExecutionContext {
return;
MetaData oldMetaData = mo.getMetaData();
- if( oldMetaData == null || !(oldMetaData instanceof MatrixFormatMetaData) )
+ if( oldMetaData == null || !(oldMetaData instanceof MetaDataFormat) )
throw new DMLRuntimeException("Metadata not available");
MatrixCharacteristics mc = new MatrixCharacteristics((long)nrows, (long)ncols,
(int) mo.getNumRowsPerBlock(), (int)mo.getNumColumnsPerBlock());
- mo.setMetaData(new MatrixFormatMetaData(mc,
- ((MatrixFormatMetaData)oldMetaData).getOutputInfo(),
- ((MatrixFormatMetaData)oldMetaData).getInputInfo()));
+ mo.setMetaData(new MetaDataFormat(mc,
+ ((MetaDataFormat)oldMetaData).getOutputInfo(),
+ ((MetaDataFormat)oldMetaData).getInputInfo()));
}
/**
@@ -611,28 +609,22 @@ public class ExecutionContext {
public void cleanupMatrixObject(MatrixObject mo)
throws DMLRuntimeException
{
- try
- {
- if ( mo.isCleanupEnabled() )
- {
- //compute ref count only if matrix cleanup actually necessary
- if ( !getVariables().hasReferences(mo) ) {
- //clean cached data
- mo.clearData();
- if( mo.isHDFSFileExists() )
- {
- //clean hdfs data
- String fpath = mo.getFileName();
- if (fpath != null) {
- MapReduceTool.deleteFileIfExistOnHDFS(fpath);
- MapReduceTool.deleteFileIfExistOnHDFS(fpath + ".mtd");
- }
- }
+ //early abort w/o scan of symbol table if no cleanup required
+ boolean fileExists = (mo.isHDFSFileExists() && mo.getFileName() != null);
+ if( !CacheableData.isCachingActive() && !fileExists )
+ return;
+
+ try {
+ //compute ref count only if matrix cleanup actually necessary
+ if ( mo.isCleanupEnabled() && !getVariables().hasReferences(mo) ) {
+ mo.clearData(); //clean cached data
+ if( fileExists ) {
+ MapReduceTool.deleteFileIfExistOnHDFS(mo.getFileName());
+ MapReduceTool.deleteFileIfExistOnHDFS(mo.getFileName()+".mtd");
}
}
}
- catch(Exception ex)
- {
+ catch(Exception ex) {
throw new DMLRuntimeException(ex);
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitioner.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitioner.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitioner.java
index d836e18..69d2a70 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitioner.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitioner.java
@@ -29,7 +29,7 @@ import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
@@ -102,7 +102,7 @@ public abstract class DataPartitioner
return in;
//analyze input matrix object
- MatrixFormatMetaData meta = (MatrixFormatMetaData)in.getMetaData();
+ MetaDataFormat meta = (MetaDataFormat)in.getMetaData();
MatrixCharacteristics mc = meta.getMatrixCharacteristics();
InputInfo ii = meta.getInputInfo();
OutputInfo oi = meta.getOutputInfo();
@@ -167,7 +167,7 @@ public abstract class DataPartitioner
mcNew.setNonZeros( nonZeros );
if( convertBlock2Cell )
ii = InputInfo.BinaryCellInputInfo;
- MatrixFormatMetaData metaNew = new MatrixFormatMetaData(mcNew,oi,ii);
+ MetaDataFormat metaNew = new MetaDataFormat(mcNew,oi,ii);
out.setMetaData(metaNew);
return out;
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
index 0effc7f..860c7b6 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
@@ -84,7 +84,7 @@ import org.apache.sysml.runtime.instructions.gpu.GPUInstruction;
import org.apache.sysml.runtime.instructions.mr.MRInstruction;
import org.apache.sysml.runtime.instructions.spark.SPInstruction;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
@@ -878,7 +878,7 @@ public class ProgramConverter
break;
case MATRIX:
MatrixObject mo = (MatrixObject) dat;
- MatrixFormatMetaData md = (MatrixFormatMetaData) dat.getMetaData();
+ MetaDataFormat md = (MetaDataFormat) dat.getMetaData();
MatrixCharacteristics mc = md.getMatrixCharacteristics();
value = mo.getFileName();
PartitionFormat partFormat = (mo.getPartitionFormat()!=null) ? new PartitionFormat(
@@ -1750,7 +1750,7 @@ public class ProgramConverter
PartitionFormat partFormat = PartitionFormat.valueOf( st.nextToken() );
UpdateType inplace = UpdateType.valueOf( st.nextToken() );
MatrixCharacteristics mc = new MatrixCharacteristics(rows, cols, brows, bcols, nnz);
- MatrixFormatMetaData md = new MatrixFormatMetaData( mc, oin, iin );
+ MetaDataFormat md = new MetaDataFormat( mc, oin, iin );
mo.setMetaData( md );
mo.setVarName( name );
if( partFormat._dpf != PDataPartitionFormat.NONE )
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalFile.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalFile.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalFile.java
index cd0b3b7..253883b 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalFile.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalFile.java
@@ -51,7 +51,7 @@ import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
import org.apache.sysml.runtime.controlprogram.parfor.util.StagingFileUtils;
import org.apache.sysml.runtime.io.IOUtilFunctions;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.IJV;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
@@ -157,7 +157,7 @@ public class ResultMergeLocalFile extends ResultMerge
{
String varName = _output.getVarName();
ValueType vt = _output.getValueType();
- MatrixFormatMetaData metadata = (MatrixFormatMetaData) _output.getMetaData();
+ MetaDataFormat metadata = (MetaDataFormat) _output.getMetaData();
MatrixObject moNew = new MatrixObject( vt, _outputFName );
moNew.setVarName( varName.contains(NAME_SUFFIX) ? varName : varName+NAME_SUFFIX );
@@ -170,7 +170,7 @@ public class ResultMergeLocalFile extends ResultMerge
MatrixCharacteristics mc = new MatrixCharacteristics(mcOld.getRows(),mcOld.getCols(),
mcOld.getRowsPerBlock(),mcOld.getColsPerBlock());
mc.setNonZeros( computeNonZeros(output, inMO) );
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,oiOld,iiOld);
+ MetaDataFormat meta = new MetaDataFormat(mc,oiOld,iiOld);
moNew.setMetaData( meta );
return moNew;
@@ -179,7 +179,7 @@ public class ResultMergeLocalFile extends ResultMerge
private void merge( String fnameNew, MatrixObject outMo, ArrayList<MatrixObject> inMO )
throws DMLRuntimeException
{
- OutputInfo oi = ((MatrixFormatMetaData)outMo.getMetaData()).getOutputInfo();
+ OutputInfo oi = ((MetaDataFormat)outMo.getMetaData()).getOutputInfo();
boolean withCompare = ( outMo.getNnz() != 0 ); //if nnz exist or unknown (-1)
if( oi == OutputInfo.TextCellOutputInfo )
@@ -295,7 +295,7 @@ public class ResultMergeLocalFile extends ResultMerge
}
//Step 2) read blocks, consolidate, and write to HDFS
- createTextCellResultFile(fnameStaging, fnameStagingCompare, fnameNew, (MatrixFormatMetaData)outMo.getMetaData(), true);
+ createTextCellResultFile(fnameStaging, fnameStagingCompare, fnameNew, (MetaDataFormat)outMo.getMetaData(), true);
}
catch(Exception ex)
{
@@ -390,7 +390,7 @@ public class ResultMergeLocalFile extends ResultMerge
}
//Step 2) read blocks, consolidate, and write to HDFS
- createBinaryCellResultFile(fnameStaging, fnameStagingCompare, fnameNew, (MatrixFormatMetaData)outMo.getMetaData(), true);
+ createBinaryCellResultFile(fnameStaging, fnameStagingCompare, fnameNew, (MetaDataFormat)outMo.getMetaData(), true);
}
catch(Exception ex)
{
@@ -420,7 +420,7 @@ public class ResultMergeLocalFile extends ResultMerge
}
//Step 2) read blocks, consolidate, and write to HDFS
- createBinaryBlockResultFile(fnameStaging, null, fnameNew, (MatrixFormatMetaData)outMo.getMetaData(), false);
+ createBinaryBlockResultFile(fnameStaging, null, fnameNew, (MetaDataFormat)outMo.getMetaData(), false);
}
catch(Exception ex)
{
@@ -454,7 +454,7 @@ public class ResultMergeLocalFile extends ResultMerge
}
//Step 2) read blocks, consolidate, and write to HDFS
- createBinaryBlockResultFile(fnameStaging, fnameStagingCompare, fnameNew, (MatrixFormatMetaData)outMo.getMetaData(), true);
+ createBinaryBlockResultFile(fnameStaging, fnameStagingCompare, fnameNew, (MetaDataFormat)outMo.getMetaData(), true);
}
catch(Exception ex)
{
@@ -644,7 +644,7 @@ public class ResultMergeLocalFile extends ResultMerge
}
@SuppressWarnings("deprecation")
- private void createBinaryBlockResultFile( String fnameStaging, String fnameStagingCompare, String fnameNew, MatrixFormatMetaData metadata, boolean withCompare )
+ private void createBinaryBlockResultFile( String fnameStaging, String fnameStagingCompare, String fnameNew, MetaDataFormat metadata, boolean withCompare )
throws IOException, DMLRuntimeException
{
JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
@@ -740,7 +740,7 @@ public class ResultMergeLocalFile extends ResultMerge
}
}
- private void createTextCellResultFile( String fnameStaging, String fnameStagingCompare, String fnameNew, MatrixFormatMetaData metadata, boolean withCompare )
+ private void createTextCellResultFile( String fnameStaging, String fnameStagingCompare, String fnameNew, MetaDataFormat metadata, boolean withCompare )
throws IOException, DMLRuntimeException
{
JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
@@ -877,7 +877,7 @@ public class ResultMergeLocalFile extends ResultMerge
}
@SuppressWarnings("deprecation")
- private void createBinaryCellResultFile( String fnameStaging, String fnameStagingCompare, String fnameNew, MatrixFormatMetaData metadata, boolean withCompare )
+ private void createBinaryCellResultFile( String fnameStaging, String fnameStagingCompare, String fnameNew, MetaDataFormat metadata, boolean withCompare )
throws IOException, DMLRuntimeException
{
JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalMemory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalMemory.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalMemory.java
index 638a845..5fcfb26 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalMemory.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalMemory.java
@@ -27,7 +27,7 @@ import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
@@ -228,7 +228,7 @@ public class ResultMergeLocalMemory extends ResultMerge
{
String varName = _output.getVarName();
ValueType vt = _output.getValueType();
- MatrixFormatMetaData metadata = (MatrixFormatMetaData) _output.getMetaData();
+ MetaDataFormat metadata = (MetaDataFormat) _output.getMetaData();
MatrixObject moNew = new MatrixObject( vt, _outputFName );
moNew.setVarName( varName.contains(NAME_SUFFIX) ? varName : varName+NAME_SUFFIX );
@@ -241,7 +241,7 @@ public class ResultMergeLocalMemory extends ResultMerge
MatrixCharacteristics mc = new MatrixCharacteristics(mcOld.getRows(),mcOld.getCols(),
mcOld.getRowsPerBlock(),mcOld.getColsPerBlock());
mc.setNonZeros(data.getNonZeros());
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,oiOld,iiOld);
+ MetaDataFormat meta = new MetaDataFormat(mc,oiOld,iiOld);
moNew.setMetaData( meta );
//adjust dense/sparse representation
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java
index 909d0d0..76cbd40 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java
@@ -41,7 +41,7 @@ import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
import org.apache.sysml.runtime.controlprogram.parfor.util.StagingFileUtils;
import org.apache.sysml.runtime.io.IOUtilFunctions;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixCell;
@@ -123,7 +123,7 @@ public class ResultMergeRemoteMR extends ResultMerge
_output.exportData();
//actual merge
- MatrixFormatMetaData metadata = (MatrixFormatMetaData) _output.getMetaData();
+ MetaDataFormat metadata = (MetaDataFormat) _output.getMetaData();
MatrixCharacteristics mcOld = metadata.getMatrixCharacteristics();
String fnameCompare = _output.getFileName();
@@ -145,7 +145,7 @@ public class ResultMergeRemoteMR extends ResultMerge
MatrixCharacteristics mc = new MatrixCharacteristics(mcOld.getRows(),mcOld.getCols(),
mcOld.getRowsPerBlock(),mcOld.getColsPerBlock());
mc.setNonZeros( computeNonZeros(_output, inMO) );
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,oiOld,iiOld);
+ MetaDataFormat meta = new MetaDataFormat(mc,oiOld,iiOld);
moNew.setMetaData( meta );
}
else
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSpark.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSpark.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSpark.java
index 95dbb8b..9accfea 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSpark.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSpark.java
@@ -38,7 +38,7 @@ import org.apache.sysml.runtime.instructions.spark.data.RDDObject;
import org.apache.sysml.runtime.instructions.spark.functions.CopyBlockPairFunction;
import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
@@ -86,7 +86,7 @@ public class ResultMergeRemoteSpark extends ResultMerge
if( _inputs != null && _inputs.length>0 )
{
//prepare compare
- MatrixFormatMetaData metadata = (MatrixFormatMetaData) _output.getMetaData();
+ MetaDataFormat metadata = (MetaDataFormat) _output.getMetaData();
MatrixCharacteristics mcOld = metadata.getMatrixCharacteristics();
MatrixObject compare = (mcOld.getNonZeros()==0) ? null : _output;
@@ -104,7 +104,7 @@ public class ResultMergeRemoteSpark extends ResultMerge
MatrixCharacteristics mc = new MatrixCharacteristics(mcOld.getRows(),mcOld.getCols(),
mcOld.getRowsPerBlock(),mcOld.getColsPerBlock());
mc.setNonZeros( computeNonZeros(_output, Arrays.asList(_inputs)) );
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,oiOld,iiOld);
+ MetaDataFormat meta = new MetaDataFormat(mc,oiOld,iiOld);
moNew.setMetaData( meta );
moNew.setRDDHandle( ro );
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
index 0d0c52e..1294809 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
@@ -94,7 +94,7 @@ import org.apache.sysml.runtime.instructions.cp.FunctionCallCPInstruction;
import org.apache.sysml.runtime.instructions.gpu.context.GPUContextPool;
import org.apache.sysml.runtime.instructions.spark.data.RDDObject;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
import org.apache.sysml.runtime.matrix.data.SparseRowVector;
@@ -2247,7 +2247,7 @@ public class OptimizerRuleBased extends Optimizer
else
{
MatrixObject mo = (MatrixObject)dat;
- MatrixFormatMetaData meta = (MatrixFormatMetaData) mo.getMetaData();
+ MetaDataFormat meta = (MetaDataFormat) mo.getMetaData();
OutputInfo oi = meta.getOutputInfo();
long nnz = meta.getMatrixCharacteristics().getNonZeros();
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/instructions/MRJobInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/MRJobInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/MRJobInstruction.java
index 3dcad83..eceebba 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/MRJobInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/MRJobInstruction.java
@@ -40,10 +40,9 @@ import org.apache.sysml.runtime.controlprogram.parfor.ProgramConverter;
import org.apache.sysml.runtime.instructions.cp.Data;
import org.apache.sysml.runtime.matrix.JobReturn;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixDimensionsMetaData;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
+import org.apache.sysml.runtime.matrix.MetaDataNumItemsByEachReducer;
import org.apache.sysml.runtime.matrix.data.InputInfo;
-import org.apache.sysml.runtime.matrix.data.NumItemsByEachReducerMetaData;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
import org.apache.sysml.runtime.util.UtilFunctions;
@@ -895,10 +894,10 @@ public class MRJobInstruction extends Instruction
clens[i] = mc.getCols();
brlens[i] = mc.getRowsPerBlock();
bclens[i] = mc.getColsPerBlock();
- if ( inputMatrices[i].getMetaData() instanceof MatrixFormatMetaData ) {
- inputInfos[i] = ((MatrixFormatMetaData) inputMatrices[i].getMetaData()).getInputInfo();
+ if ( inputMatrices[i].getMetaData() instanceof MetaDataFormat ) {
+ inputInfos[i] = ((MetaDataFormat) inputMatrices[i].getMetaData()).getInputInfo();
}
- else if (inputMatrices[i].getMetaData() instanceof NumItemsByEachReducerMetaData ) {
+ else if (inputMatrices[i].getMetaData() instanceof MetaDataNumItemsByEachReducer ) {
inputInfos[i] = InputInfo.InputInfoForSortOutput;
inputInfos[i].metadata = inputMatrices[i].getMetaData();
}
@@ -923,7 +922,7 @@ public class MRJobInstruction extends Instruction
// Populate information
for(int i=0; i < outputVars.length; i++) {
outputs[i] = outputMatrices[i].getFileName();
- MatrixFormatMetaData md = (MatrixFormatMetaData) outputMatrices[i].getMetaData();
+ MetaDataFormat md = (MetaDataFormat) outputMatrices[i].getMetaData();
outputInfos[i] = md.getOutputInfo();
}
}
@@ -1294,7 +1293,7 @@ public class MRJobInstruction extends Instruction
/* Populate returned stats into symbol table of matrices */
for ( int index=0; index < jb.getMetaData().length; index++) {
String varname = getOutputVars()[index];
- MatrixCharacteristics mc = ((MatrixDimensionsMetaData)jb.getMetaData(index)).getMatrixCharacteristics();
+ MatrixCharacteristics mc = jb.getMetaData(index).getMatrixCharacteristics();
ec.getVariable(varname).updateMatrixCharacteristics(mc);
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/instructions/cp/DataPartitionCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/DataPartitionCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/DataPartitionCPInstruction.java
index f52b8f4..6bb6fcd 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/DataPartitionCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/DataPartitionCPInstruction.java
@@ -30,7 +30,7 @@ import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysml.runtime.instructions.InstructionUtils;
import org.apache.sysml.runtime.io.WriterBinaryBlock;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
@@ -85,7 +85,7 @@ public class DataPartitionCPInstruction extends UnaryCPInstruction {
//ensure correctness of output characteristics (required if input unknown during compile and no recompile)
MatrixCharacteristics mc = new MatrixCharacteristics(moIn.getNumRows(), moIn.getNumColumns(), (int)moIn.getNumRowsPerBlock(), (int)moIn.getNumColumnsPerBlock(), moIn.getNnz());
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
+ MetaDataFormat meta = new MetaDataFormat(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
moOut.setMetaData(meta);
}
catch(Exception ex)
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/instructions/cp/QuantilePickCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/QuantilePickCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/QuantilePickCPInstruction.java
index cace173..1ad6054 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/QuantilePickCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/QuantilePickCPInstruction.java
@@ -28,8 +28,8 @@ import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysml.runtime.instructions.InstructionUtils;
import org.apache.sysml.runtime.matrix.MetaData;
+import org.apache.sysml.runtime.matrix.MetaDataNumItemsByEachReducer;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
-import org.apache.sysml.runtime.matrix.data.NumItemsByEachReducerMetaData;
import org.apache.sysml.runtime.matrix.operators.Operator;
import org.apache.sysml.runtime.util.MapReduceTool;
import org.apache.sysml.runtime.util.UtilFunctions;
@@ -130,7 +130,7 @@ public class QuantilePickCPInstruction extends BinaryCPInstruction {
if ( mdata != null ) {
try {
- double picked = MapReduceTool.pickValue(fname, (NumItemsByEachReducerMetaData) mdata, pickindex.getDoubleValue());
+ double picked = MapReduceTool.pickValue(fname, (MetaDataNumItemsByEachReducer) mdata, pickindex.getDoubleValue());
ec.setVariable(output.getName(), new DoubleObject(picked));
} catch (Exception e ) {
throw new DMLRuntimeException(e);
@@ -158,7 +158,7 @@ public class QuantilePickCPInstruction extends BinaryCPInstruction {
if ( mdata1 != null ) {
try {
- double median = MapReduceTool.median(fname1, (NumItemsByEachReducerMetaData) mdata1);
+ double median = MapReduceTool.median(fname1, (MetaDataNumItemsByEachReducer) mdata1);
ec.setVariable(output.getName(), new DoubleObject(median));
} catch (Exception e ) {
throw new DMLRuntimeException(e);
@@ -186,13 +186,13 @@ public class QuantilePickCPInstruction extends BinaryCPInstruction {
double[] q25 = null;
double[] q75 = null;
try {
- q25 = MapReduceTool.pickValueWeight(inputMatrix.getFileName(), (NumItemsByEachReducerMetaData) inputMatrix.getMetaData(), 0.25, false);
- q75 = MapReduceTool.pickValueWeight(inputMatrix.getFileName(), (NumItemsByEachReducerMetaData) inputMatrix.getMetaData(), 0.75, false);
+ q25 = MapReduceTool.pickValueWeight(inputMatrix.getFileName(), (MetaDataNumItemsByEachReducer) inputMatrix.getMetaData(), 0.25, false);
+ q75 = MapReduceTool.pickValueWeight(inputMatrix.getFileName(), (MetaDataNumItemsByEachReducer) inputMatrix.getMetaData(), 0.75, false);
} catch (IOException e1) {
throw new DMLRuntimeException(e1);
}
- double sumwt = UtilFunctions.getTotalLength((NumItemsByEachReducerMetaData) ec.getMetaData(input1.getName()));
+ double sumwt = UtilFunctions.getTotalLength((MetaDataNumItemsByEachReducer) ec.getMetaData(input1.getName()));
double q25d = sumwt*0.25;
double q75d = sumwt*0.75;
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
index 1c0f9e6..2239919 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
@@ -46,7 +46,7 @@ import org.apache.sysml.runtime.io.IOUtilFunctions;
import org.apache.sysml.runtime.io.WriterMatrixMarket;
import org.apache.sysml.runtime.io.WriterTextCSV;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.MetaData;
import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
import org.apache.sysml.runtime.matrix.data.FileFormatProperties;
@@ -360,7 +360,7 @@ public class VariableCPInstruction extends CPInstruction {
else {
throw new DMLRuntimeException("Invalid number of operands in createvar instruction: " + str);
}
- MatrixFormatMetaData iimd = new MatrixFormatMetaData(mc, oi, ii);
+ MetaDataFormat iimd = new MetaDataFormat(mc, oi, ii);
UpdateType updateType = UpdateType.COPY;
if ( parts.length >= 12 )
updateType = UpdateType.valueOf(parts[11].toUpperCase());
@@ -489,16 +489,12 @@ public class VariableCPInstruction extends CPInstruction {
//create new variable for symbol table and cache
//(existing objects gets cleared through rmvar instructions)
String fname = getInput2().getName();
-
// check if unique filename needs to be generated
- boolean overrideFileName = ((BooleanObject) ec.getScalarInput(getInput3().getName(), getInput3().getValueType(), true)).getBooleanValue();
- if ( overrideFileName ) {
+ if( Boolean.parseBoolean(getInput3().getName()) )
fname = fname + "_" + _uniqueVarID.getNextID();
- }
MatrixObject mobj = new MatrixObject(getInput1().getValueType(), fname );
mobj.setVarName(getInput1().getName());
- mobj.setDataType(DataType.MATRIX);
//clone meta data because it is updated on copy-on-write, otherwise there
//is potential for hidden side effects between variables.
mobj.setMetaData((MetaData)metadata.clone());
@@ -512,7 +508,6 @@ public class VariableCPInstruction extends CPInstruction {
String fname = getInput2().getName();
FrameObject fobj = new FrameObject(fname);
fobj.setVarName(getInput1().getName());
- fobj.setDataType(DataType.FRAME);
fobj.setMetaData((MetaData)metadata.clone());
fobj.setFileFormatProperties(_formatProperties);
if( _schema != null )
@@ -520,8 +515,8 @@ public class VariableCPInstruction extends CPInstruction {
ec.setVariable(getInput1().getName(), fobj);
}
else if ( getInput1().getDataType() == DataType.SCALAR ){
- ScalarObject sobj = null;
- ec.setScalarOutput(getInput1().getName(), sobj);
+ //created variable not called for scalars
+ ec.setScalarOutput(getInput1().getName(), null);
}
else {
throw new DMLRuntimeException("Unexpected data type: " + getInput1().getDataType());
@@ -862,8 +857,8 @@ public class VariableCPInstruction extends CPInstruction {
}
else {
try {
- OutputInfo oi = ((MatrixFormatMetaData)mo.getMetaData()).getOutputInfo();
- MatrixCharacteristics mc = ((MatrixFormatMetaData)mo.getMetaData()).getMatrixCharacteristics();
+ OutputInfo oi = ((MetaDataFormat)mo.getMetaData()).getOutputInfo();
+ MatrixCharacteristics mc = ((MetaDataFormat)mo.getMetaData()).getMatrixCharacteristics();
if(oi == OutputInfo.CSVOutputInfo) {
WriterTextCSV writer = new WriterTextCSV((CSVFileFormatProperties)_formatProperties);
writer.addHeaderToCSV(mo.getFileName(), fname, mc.getRows(), mc.getCols());
@@ -901,7 +896,7 @@ public class VariableCPInstruction extends CPInstruction {
mo.exportData(fname, outFmt);
}
else {
- OutputInfo oi = ((MatrixFormatMetaData)mo.getMetaData()).getOutputInfo();
+ OutputInfo oi = ((MetaDataFormat)mo.getMetaData()).getOutputInfo();
MatrixCharacteristics mc = mo.getMatrixCharacteristics();
if(oi == OutputInfo.TextCellOutputInfo) {
try {
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/instructions/cpfile/MatrixIndexingCPFileInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cpfile/MatrixIndexingCPFileInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cpfile/MatrixIndexingCPFileInstruction.java
index acfd4d4..0b35837 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cpfile/MatrixIndexingCPFileInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cpfile/MatrixIndexingCPFileInstruction.java
@@ -29,7 +29,7 @@ import org.apache.sysml.runtime.instructions.InstructionUtils;
import org.apache.sysml.runtime.instructions.cp.CPOperand;
import org.apache.sysml.runtime.instructions.cp.IndexingCPInstruction;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.operators.Operator;
import org.apache.sysml.runtime.matrix.operators.SimpleOperator;
@@ -92,7 +92,7 @@ public final class MatrixIndexingCPFileInstruction extends IndexingCPInstruction
if( mo.isPartitioned() && opcode.equalsIgnoreCase(RightIndex.OPCODE) )
{
- MatrixFormatMetaData meta = (MatrixFormatMetaData)mo.getMetaData();
+ MetaDataFormat meta = (MetaDataFormat)mo.getMetaData();
MatrixCharacteristics mc = meta.getMatrixCharacteristics();
String pfname = mo.getPartitionFileName( ixrange, mc.getRowsPerBlock(), mc.getColsPerBlock());
@@ -123,7 +123,7 @@ public final class MatrixIndexingCPFileInstruction extends IndexingCPInstruction
throw new DMLRuntimeException("Unsupported partition format for CP_FILE "+RightIndex.OPCODE+": "+ mo.getPartitionFormat());
}
- MatrixFormatMetaData metaNew = new MatrixFormatMetaData(mcNew,meta.getOutputInfo(),meta.getInputInfo());
+ MetaDataFormat metaNew = new MetaDataFormat(mcNew,meta.getOutputInfo(),meta.getInputInfo());
mobj.setMetaData(metaNew);
//put output object into symbol table
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java
index a85b7ad..fed3ad5 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java
@@ -59,7 +59,7 @@ import org.apache.sysml.runtime.instructions.cp.ParameterizedBuiltinCPInstructio
import org.apache.sysml.runtime.io.IOUtilFunctions;
import org.apache.sysml.runtime.io.MatrixWriter;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixCell;
@@ -161,7 +161,7 @@ public class ParameterizedBuiltinCPFileInstruction extends ParameterizedBuiltinC
//initial setup
String fnameOld = _src.getFileName();
String fnameNew = _out.getFileName();
- InputInfo ii = ((MatrixFormatMetaData)_src.getMetaData()).getInputInfo();
+ InputInfo ii = ((MetaDataFormat)_src.getMetaData()).getInputInfo();
MatrixCharacteristics mc = _src.getMatrixCharacteristics();
String stagingDir = LocalFileUtils.getUniqueWorkingDir(LocalFileUtils.CATEGORY_WORK);
@@ -228,7 +228,7 @@ public class ParameterizedBuiltinCPFileInstruction extends ParameterizedBuiltinC
String varName = out.getVarName();
String fName = out.getFileName();
ValueType vt = src.getValueType();
- MatrixFormatMetaData metadata = (MatrixFormatMetaData) src.getMetaData();
+ MetaDataFormat metadata = (MetaDataFormat) src.getMetaData();
MatrixObject moNew = new MatrixObject( vt, fName );
moNew.setVarName( varName );
@@ -253,7 +253,7 @@ public class ParameterizedBuiltinCPFileInstruction extends ParameterizedBuiltinC
InputInfo iiOld = metadata.getInputInfo();
MatrixCharacteristics mc = new MatrixCharacteristics( rows, cols, mcOld.getRowsPerBlock(),
mcOld.getColsPerBlock(), mcOld.getNonZeros());
- MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,oiOld,iiOld);
+ MetaDataFormat meta = new MetaDataFormat(mc,oiOld,iiOld);
moNew.setMetaData( meta );
return moNew;
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java
index 43dd327..fb80deb 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java
@@ -35,7 +35,7 @@ import org.apache.sysml.runtime.instructions.cp.CPOperand;
import org.apache.sysml.runtime.instructions.spark.utils.FrameRDDConverterUtils;
import org.apache.sysml.runtime.instructions.spark.utils.RDDConverterUtils;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
@@ -94,7 +94,7 @@ public class CSVReblockSPInstruction extends UnarySPInstruction {
//sanity check input info
CacheableData<?> obj = sec.getCacheableData(input1.getName());
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) obj.getMetaData();
+ MetaDataFormat iimd = (MetaDataFormat) obj.getMetaData();
if (iimd.getInputInfo() != InputInfo.CSVInputInfo) {
throw new DMLRuntimeException("The given InputInfo is not implemented for "
+ "CSVReblockSPInstruction:" + iimd.getInputInfo());
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/instructions/spark/ConvolutionSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/ConvolutionSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/ConvolutionSPInstruction.java
index 4ea4f0b..874371d 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/ConvolutionSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/ConvolutionSPInstruction.java
@@ -37,7 +37,7 @@ import org.apache.sysml.runtime.instructions.spark.data.LazyIterableIterator;
import org.apache.sysml.runtime.instructions.spark.functions.ExtractBlockForBinaryReblock;
import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.ConvolutionParameters;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.LibMatrixDNN;
@@ -291,7 +291,7 @@ public class ConvolutionSPInstruction extends UnarySPInstruction {
throw new DMLRuntimeException("The current operator doesnot support large outputs.");
}
sec.setMetaData(output.getName(),
- new MatrixFormatMetaData(new MatrixCharacteristics(mcRdd.getRows(), numCols, numRowsPerBlock, (int)numCols, nnz), OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo));
+ new MetaDataFormat(new MatrixCharacteristics(mcRdd.getRows(), numCols, numRowsPerBlock, (int)numCols, nnz), OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo));
}
else {
throw new DMLRuntimeException("Not implemented: " + instOpcode);
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/instructions/spark/ReblockSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/ReblockSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/ReblockSPInstruction.java
index 0839c5b..d529084 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/ReblockSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/ReblockSPInstruction.java
@@ -37,7 +37,7 @@ import org.apache.sysml.runtime.instructions.spark.utils.FrameRDDConverterUtils;
import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
import org.apache.sysml.runtime.instructions.spark.utils.RDDConverterUtils;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.MetaDataFormat;
import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.InputInfo;
@@ -92,7 +92,7 @@ public class ReblockSPInstruction extends UnarySPInstruction {
mcOut.set(mc.getRows(), mc.getCols(), brlen, bclen, mc.getNonZeros());
//get the source format form the meta data
- MatrixFormatMetaData iimd = (MatrixFormatMetaData) obj.getMetaData();
+ MetaDataFormat iimd = (MetaDataFormat) obj.getMetaData();
if(iimd == null)
throw new DMLRuntimeException("Error: Metadata not found");
InputInfo iinfo = iimd.getInputInfo();