You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2016/08/22 21:38:54 UTC
incubator-systemml git commit: [SYSTEMML-864] Fix nrow/ncol
operations over csv frame w/ unknown size
Repository: incubator-systemml
Updated Branches:
refs/heads/master 0827a4638 -> 7c12992af
[SYSTEMML-864] Fix nrow/ncol operations over csv frame w/ unknown size
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/7c12992a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/7c12992a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/7c12992a
Branch: refs/heads/master
Commit: 7c12992af95b013b9380acd3a6843ee89fae7a3e
Parents: 0827a46
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Mon Aug 22 10:51:50 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Mon Aug 22 14:38:35 2016 -0700
----------------------------------------------------------------------
.../org/apache/sysml/hops/OptimizerUtils.java | 9 +++++++++
.../cp/AggregateUnaryCPInstruction.java | 20 +++++++++++++-------
2 files changed, 22 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7c12992a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
index 8e53e96..52ee378 100644
--- a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
+++ b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
@@ -594,6 +594,15 @@ public class OptimizerUtils
*
* @return
*/
+ public static boolean isHadoopExecutionMode() {
+ return ( DMLScript.rtplatform == RUNTIME_PLATFORM.HADOOP
+ || DMLScript.rtplatform == RUNTIME_PLATFORM.HYBRID);
+ }
+
+ /**
+ *
+ * @return
+ */
public static boolean isHybridExecutionMode() {
return ( DMLScript.rtplatform == RUNTIME_PLATFORM.HYBRID
|| DMLScript.rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK );
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7c12992a/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java
index 24b2438..bafaeb2 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java
@@ -21,9 +21,10 @@ package org.apache.sysml.runtime.instructions.cp;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.runtime.DMLRuntimeException;
-import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.caching.CacheableData;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysml.runtime.functionobjects.Builtin;
import org.apache.sysml.runtime.instructions.InstructionUtils;
@@ -101,13 +102,18 @@ public class AggregateUnaryCPInstruction extends UnaryCPInstruction
//Note: check on matrix characteristics to cover incorrect length (-1*-1 -> 1)
if( !mc.dimsKnown() ) //invalid nrow/ncol/length
{
- if( DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE )
+ if( DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE
+ || (input1.getDataType() == DataType.FRAME && OptimizerUtils.isHadoopExecutionMode()) )
{
- //read the input data and explicitly refresh input data
- MatrixObject mo = (MatrixObject)ec.getVariable(input1.getName());
- mo.acquireRead();
- mo.refreshMetaData();
- mo.release();
+ if( OptimizerUtils.isHadoopExecutionMode() ) {
+ LOG.warn("Reading csv input frame of unkown size into memory for '"+opcode+"'.");
+ }
+
+ //read the input matrix/frame and explicitly refresh meta data
+ CacheableData<?> obj = ec.getCacheableData(input1.getName());
+ obj.acquireRead();
+ obj.refreshMetaData();
+ obj.release();
//update meta data information
mc = ec.getMatrixCharacteristics(input1.getName());