You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by du...@apache.org on 2016/10/01 22:46:08 UTC
incubator-systemml git commit: [SYSTEMML-995] Determine Frame Format
If Needed
Repository: incubator-systemml
Updated Branches:
refs/heads/master 85a8746a7 -> d9de9c792
[SYSTEMML-995] Determine Frame Format If Needed
If `MLContextConversionUtil.dataFrameToFrameObject` receives a
DataFrame *without* any frame metadata, a new `FrameMetadata` will be
created with an empty `FrameFormat`, and so the subsequent
`isDataFrameWithIDColumn` function will always return false
([line 360](https://github.com/apache/incubator-systemml/blob/master/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java#L360).
We should just create a new function similar to
`determineMatrixFormatIfNeeded` for frames, and call it before the
`isDataFrameWithIDColumn` function, as is done for DataFrame-matrix
conversions ([line 412](https://github.com/apache/incubator-systemml/blob/master/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java#L412).
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/d9de9c79
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/d9de9c79
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/d9de9c79
Branch: refs/heads/master
Commit: d9de9c7927915e411692df79c90afd0eaa6323e0
Parents: 85a8746
Author: Mike Dusenberry <mw...@us.ibm.com>
Authored: Sat Oct 1 15:40:34 2016 -0700
Committer: Mike Dusenberry <mw...@us.ibm.com>
Committed: Sat Oct 1 15:44:13 2016 -0700
----------------------------------------------------------------------
.../api/mlcontext/MLContextConversionUtil.java | 28 +++++++++++++++++++
.../mlcontext/DataFrameVectorScriptTest.java | 29 +++++++++++++-------
2 files changed, 47 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d9de9c79/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java b/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java
index 53564d9..ab22775 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java
@@ -357,6 +357,7 @@ public class MLContextConversionUtil {
//setup meta data and java spark context
if (frameMetadata == null)
frameMetadata = new FrameMetadata();
+ determineFrameFormatIfNeeded(dataFrame, frameMetadata);
boolean containsID = isDataFrameWithIDColumn(frameMetadata);
JavaSparkContext javaSparkContext = MLContextUtil
.getJavaSparkContext((MLContext) MLContextProxy.getActiveMLContextForAPI());
@@ -491,6 +492,33 @@ public class MLContextConversionUtil {
}
/**
+ * If the FrameFormat of the DataFrame has not been explicitly specified,
+ * attempt to determine the proper FrameFormat.
+ *
+ * @param dataFrame
+ * the Spark {@code DataFrame}
+ * @param frameMetadata
+ * the frame metadata, if available
+ */
+ public static void determineFrameFormatIfNeeded(DataFrame dataFrame, FrameMetadata frameMetadata) {
+ FrameFormat frameFormat = frameMetadata.getFrameFormat();
+ if (frameFormat != null) {
+ return;
+ }
+
+ StructType schema = dataFrame.schema();
+ boolean hasID = false;
+ try {
+ schema.fieldIndex(RDDConverterUtils.DF_ID_COLUMN);
+ hasID = true;
+ } catch (IllegalArgumentException iae) {
+ }
+
+ FrameFormat ff = hasID ? FrameFormat.DF_WITH_INDEX : FrameFormat.DF;
+ frameMetadata.setFrameFormat(ff);
+ }
+
+ /**
* Return whether or not the DataFrame has an ID column.
*
* @param matrixMetadata
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d9de9c79/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/DataFrameVectorScriptTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/DataFrameVectorScriptTest.java b/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/DataFrameVectorScriptTest.java
index 05b8fce..a35fe95 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/DataFrameVectorScriptTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/DataFrameVectorScriptTest.java
@@ -271,24 +271,33 @@ public class DataFrameVectorScriptTest extends AutomatedTestBase
//create input data frame
DataFrame df = createDataFrame(sqlctx, mbA, containsID, schema);
- FrameMetadata meta = new FrameMetadata(containsID ? FrameFormat.DF_WITH_INDEX :
+
+ // Create full frame metadata, and empty frame metadata
+ FrameMetadata meta = new FrameMetadata(containsID ? FrameFormat.DF_WITH_INDEX :
FrameFormat.DF, mc2.getRows(), mc2.getCols());
-
+ FrameMetadata metaEmpty = new FrameMetadata();
+
//create mlcontext
ml = new MLContext(sc);
ml.setExplain(true);
- //run script and obtain result
+ //run scripts and obtain result
Script script1 = dml(
- //"Xf = read($Xffile); Xm = as.matrix(Xf); write(Xm, $Xmfile);")
"Xm = as.matrix(Xf);")
.in("Xf", df, meta).out("Xm");
- Matrix Xm = ml.execute(script1).getMatrix("Xm");
- MatrixBlock mbB = Xm.toBinaryBlockMatrix().getMatrixBlock();
-
+ Script script2 = dml(
+ "Xm = as.matrix(Xf);")
+ .in("Xf", df, metaEmpty).out("Xm"); // empty metadata
+ Matrix Xm1 = ml.execute(script1).getMatrix("Xm");
+ Matrix Xm2 = ml.execute(script2).getMatrix("Xm");
+ MatrixBlock mbB1 = Xm1.toBinaryBlockMatrix().getMatrixBlock();
+ MatrixBlock mbB2 = Xm2.toBinaryBlockMatrix().getMatrixBlock();
+
//compare frame blocks
- double[][] B = DataConverter.convertToDoubleMatrix(mbB);
- TestUtils.compareMatrices(A, B, rows1, cols, eps);
+ double[][] B1 = DataConverter.convertToDoubleMatrix(mbB1);
+ double[][] B2 = DataConverter.convertToDoubleMatrix(mbB2);
+ TestUtils.compareMatrices(A, B1, rows1, cols, eps);
+ TestUtils.compareMatrices(A, B2, rows1, cols, eps);
}
catch( Exception ex ) {
ex.printStackTrace();
@@ -364,4 +373,4 @@ public class DataFrameVectorScriptTest extends AutomatedTestBase
JavaRDD<Row> rowRDD = sc.parallelize(list);
return sqlctx.createDataFrame(rowRDD, dfSchema);
}
-}
\ No newline at end of file
+}