You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2016/09/16 18:03:54 UTC
incubator-systemml git commit: [SYSTEMML-914] Fix old mlcontext frame
output handling (avoid cast2mtx)
Repository: incubator-systemml
Updated Branches:
refs/heads/master df090f2b1 -> 6758f9034
[SYSTEMML-914] Fix old mlcontext frame output handling (avoid cast2mtx)
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/6758f903
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/6758f903
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/6758f903
Branch: refs/heads/master
Commit: 6758f9034f2163ae8eb8d13b700594ee06963481
Parents: df090f2
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Fri Sep 16 20:02:56 2016 +0200
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Fri Sep 16 20:02:56 2016 +0200
----------------------------------------------------------------------
.../java/org/apache/sysml/api/MLContext.java | 44 ++++++--------------
.../java/org/apache/sysml/api/MLOutput.java | 4 --
2 files changed, 12 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/6758f903/src/main/java/org/apache/sysml/api/MLContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/MLContext.java b/src/main/java/org/apache/sysml/api/MLContext.java
index 1e415f1..ced3998 100644
--- a/src/main/java/org/apache/sysml/api/MLContext.java
+++ b/src/main/java/org/apache/sysml/api/MLContext.java
@@ -1489,53 +1489,34 @@ public class MLContext {
// Set active MLContext.
_activeMLContext = this;
-
if(_monitorUtils != null) {
_monitorUtils.resetMonitoringData();
}
- if(DMLScript.rtplatform == RUNTIME_PLATFORM.SPARK || DMLScript.rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK) {
-
- Map<String, JavaPairRDD<?,?>> retVal = null;
-
+ if( OptimizerUtils.isSparkExecutionMode() ) {
// Depending on whether registerInput/registerOutput was called initialize the variables
- String[] inputs; String[] outputs;
- if(_inVarnames != null) {
- inputs = _inVarnames.toArray(new String[0]);
- }
- else {
- inputs = new String[0];
- }
- if(_outVarnames != null) {
- outputs = _outVarnames.toArray(new String[0]);
- }
- else {
- outputs = new String[0];
- }
+ String[] inputs = (_inVarnames != null) ? _inVarnames.toArray(new String[0]) : new String[0];
+ String[] outputs = (_outVarnames != null) ? _outVarnames.toArray(new String[0]) : new String[0];
+ Map<String, JavaPairRDD<?,?>> retVal = (_outVarnames!=null && !_outVarnames.isEmpty()) ?
+ retVal = new HashMap<String, JavaPairRDD<?,?>>() : null;
Map<String, MatrixCharacteristics> outMetadata = new HashMap<String, MatrixCharacteristics>();
-
Map<String, String> argVals = DMLScript.createArgumentsMap(isNamedArgument, args);
// Run the DML script
ExecutionContext ec = executeUsingSimplifiedCompilationChain(dmlScriptFilePath, isFile, argVals, isPyDML, inputs, outputs, _variables, configFilePath);
+ SparkExecutionContext sec = (SparkExecutionContext) ec;
// Now collect the output
if(_outVarnames != null) {
- if(_variables == null) {
- throw new DMLRuntimeException("The symbol table returned after executing the script is empty");
- }
+ if(_variables == null)
+ throw new DMLRuntimeException("The symbol table returned after executing the script is empty");
for( String ovar : _outVarnames ) {
- if( _variables.keySet().contains(ovar) ) {
- if(retVal == null) {
- retVal = new HashMap<String, JavaPairRDD<?,?>>();
- }
- retVal.put(ovar, ((SparkExecutionContext) ec).getBinaryBlockRDDHandleForVariable(ovar));
- outMetadata.put(ovar, ec.getMatrixCharacteristics(ovar)); // For converting output to dataframe
- }
- else {
+ if( !_variables.keySet().contains(ovar) )
throw new DMLException("Error: The variable " + ovar + " is not available as output after the execution of the DMLScript.");
- }
+
+ retVal.put(ovar, sec.getRDDHandleForVariable(ovar, InputInfo.BinaryBlockInputInfo));
+ outMetadata.put(ovar, ec.getMatrixCharacteristics(ovar)); // For converting output to dataframe
}
}
@@ -1544,7 +1525,6 @@ public class MLContext {
else {
throw new DMLRuntimeException("Unsupported runtime:" + DMLScript.rtplatform.name());
}
-
}
finally {
// Remove global dml config and all thread-local configs
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/6758f903/src/main/java/org/apache/sysml/api/MLOutput.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/MLOutput.java b/src/main/java/org/apache/sysml/api/MLOutput.java
index d011104..5612392 100644
--- a/src/main/java/org/apache/sysml/api/MLOutput.java
+++ b/src/main/java/org/apache/sysml/api/MLOutput.java
@@ -159,8 +159,6 @@ public class MLOutput {
}
public JavaRDD<String> getStringFrameRDD(String varName, String format, CSVFileFormatProperties fprop ) throws DMLRuntimeException {
- //TODO MB: note that on construction of MLOutput only matrix binary blocks are passed, and
- //hence we will never find a frame binary block in the outputs.
JavaPairRDD<Long, FrameBlock> binaryRDD = getFrameBinaryBlockedRDD(varName);
MatrixCharacteristics mcIn = getMatrixCharacteristics(varName);
if(format.equals("csv")) {
@@ -176,8 +174,6 @@ public class MLOutput {
}
public DataFrame getDataFrameRDD(String varName, JavaSparkContext jsc) throws DMLRuntimeException {
- //TODO MB: note that on construction of MLOutput only matrix binary blocks are passed, and
- //hence we will never find a frame binary block in the outputs.
JavaPairRDD<Long, FrameBlock> binaryRDD = getFrameBinaryBlockedRDD(varName);
MatrixCharacteristics mcIn = getMatrixCharacteristics(varName);
return FrameRDDConverterUtils.binaryBlockToDataFrame(new SQLContext(jsc), binaryRDD, mcIn, null);