You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2016/09/17 20:33:32 UTC
incubator-systemml git commit: [SYSTEMML-929] Enable frame csv
reblock for frame-based transform
Repository: incubator-systemml
Updated Branches:
refs/heads/master 8d5f3ceab -> 062ebc67a
[SYSTEMML-929] Enable frame csv reblock for frame-based transform
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/062ebc67
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/062ebc67
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/062ebc67
Branch: refs/heads/master
Commit: 062ebc67a821e12c7e3c04eab3b5d863242a7445
Parents: 8d5f3ce
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Sat Sep 17 01:02:26 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Sat Sep 17 01:02:26 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/sysml/hops/OptimizerUtils.java | 5 +----
.../parser/ParameterizedBuiltinFunctionExpression.java | 13 ++++++++++++-
2 files changed, 13 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/062ebc67/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
index c19ad57..b0a938d 100644
--- a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
+++ b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
@@ -205,11 +205,8 @@ public class OptimizerUtils
/**
* Enables automatic csv-binary block reblock.
- *
- * TODO enable by default and remove once file-based transform completely
- * removed via frame-based transform/transformapply
*/
- public static boolean ALLOW_FRAME_CSV_REBLOCK = false;
+ public static boolean ALLOW_FRAME_CSV_REBLOCK = true;
public static long GPU_MEMORY_BUDGET = -1;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/062ebc67/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java b/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java
index 679075b..90364d7 100644
--- a/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java
+++ b/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java
@@ -25,6 +25,7 @@ import java.util.HashMap;
import java.util.HashSet;
import org.apache.sysml.hops.Hop.ParamBuiltinOp;
+import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.parser.LanguageException.LanguageErrorCodes;
@@ -298,7 +299,9 @@ public class ParameterizedBuiltinFunctionExpression extends DataIdentifier
}
// example: A = transform(data=D, txmtd="", txspec="")
- private void validateTransform(DataIdentifier output, boolean conditional) throws LanguageException {
+ private void validateTransform(DataIdentifier output, boolean conditional)
+ throws LanguageException
+ {
//validate data
checkDataType("transform", TF_FN_PARAM_DATA, DataType.FRAME, conditional);
@@ -337,6 +340,14 @@ public class ParameterizedBuiltinFunctionExpression extends DataIdentifier
raiseValidateError("Only one of '" + TF_FN_PARAM_APPLYMTD + "' or '" + TF_FN_PARAM_OUTNAMES + "' can be specified in transform().", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
}
+ // disable frame csv reblocks as transform operates directly over csv files
+ // (this is required to support both file-based transform and frame-based
+ // transform at the same time; hence, transform and frame-based transform
+ // functions over csv cannot be used in the same script; accordingly we
+ // give an appropriate warning)
+ OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = false;
+ raiseValidateError("Disable frame csv reblock to support file-based transform.", true);
+
// Output is a matrix with same dims as input
output.setDataType(DataType.MATRIX);
output.setFormatType(FormatType.CSV);