You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2016/04/10 20:51:00 UTC
[1/4] incubator-systemml git commit: [SYSTEMML-556] Robustness
jmlc/mlcontext apis (warnings non-reg in/out)
Repository: incubator-systemml
Updated Branches:
refs/heads/master 8dda34b13 -> 67e0d9d25
[SYSTEMML-556] Robustness jmlc/mlcontext apis (warnings non-reg in/out)
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/1c476dab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/1c476dab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/1c476dab
Branch: refs/heads/master
Commit: 1c476dabadba031de4936f1ba8eeb80f3ea9d1d2
Parents: 8dda34b
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Fri Apr 8 15:41:55 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Fri Apr 8 22:02:49 2016 -0700
----------------------------------------------------------------------
.../sysml/hops/rewrite/RewriteRemovePersistentReadWrite.java | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1c476dab/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemovePersistentReadWrite.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemovePersistentReadWrite.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemovePersistentReadWrite.java
index 9f5d235..b908013 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemovePersistentReadWrite.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteRemovePersistentReadWrite.java
@@ -22,6 +22,8 @@ package org.apache.sysml.hops.rewrite;
import java.util.ArrayList;
import java.util.HashSet;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.sysml.hops.DataOp;
import org.apache.sysml.hops.Hop;
import org.apache.sysml.hops.Hop.DataOpTypes;
@@ -36,6 +38,7 @@ import org.apache.sysml.hops.Hop.VisitStatus;
*/
public class RewriteRemovePersistentReadWrite extends HopRewriteRule
{
+ private static final Log LOG = LogFactory.getLog(RewriteRemovePersistentReadWrite.class.getName());
private HashSet<String> _inputs = null;
private HashSet<String> _outputs = null;
@@ -103,10 +106,14 @@ public class RewriteRemovePersistentReadWrite extends HopRewriteRule
case PERSISTENTREAD:
if( _inputs.contains(dop.getName()) )
dop.setDataOpType(DataOpTypes.TRANSIENTREAD);
+ else
+ LOG.warn("Non-registered persistent read of variable '"+dop.getName()+"' (line "+dop.getBeginLine()+").");
break;
case PERSISTENTWRITE:
if( _outputs.contains(dop.getName()) )
dop.setDataOpType(DataOpTypes.TRANSIENTWRITE);
+ else
+ LOG.warn("Non-registered persistent write of variable '"+dop.getName()+"' (line "+dop.getBeginLine()+").");
break;
default:
//do nothing
[4/4] incubator-systemml git commit: [SYSTEMML-616] Cleanup
instruction handling (stage 2, remove redundant)
Posted by mb...@apache.org.
[SYSTEMML-616] Cleanup instruction handling (stage 2, remove redundant)
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/67e0d9d2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/67e0d9d2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/67e0d9d2
Branch: refs/heads/master
Commit: 67e0d9d25da26779bc3c6617fa767b3d289c7a4a
Parents: b8152ea
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Sat Apr 9 00:42:56 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Sat Apr 9 00:42:56 2016 -0700
----------------------------------------------------------------------
.../apache/sysml/api/ml/LogisticRegression.java | 13 +-
.../sysml/api/ml/LogisticRegressionModel.java | 11 +-
.../sysml/hops/ipa/InterProceduralAnalysis.java | 1 +
.../apache/sysml/parser/DMLParseException.java | 89 -------------
.../org/apache/sysml/parser/DataExpression.java | 47 ++++---
.../org/apache/sysml/parser/Expression.java | 18 +--
.../apache/sysml/parser/IndexedIdentifier.java | 92 ++++++-------
.../apache/sysml/parser/OutputStatement.java | 37 ++----
.../org/apache/sysml/parser/ParseException.java | 123 ++++++-----------
.../java/org/apache/sysml/parser/Statement.java | 34 +++++
.../org/apache/sysml/parser/StatementBlock.java | 19 +--
.../parser/common/CustomErrorListener.java | 6 +
.../ExternalFunctionProgramBlock.java | 132 +++++--------------
.../ExternalFunctionProgramBlockCP.java | 3 +-
.../parfor/opt/PerfTestExtFunctCP.java | 3 +-
.../sysml/runtime/util/DataConverter.java | 12 +-
src/main/java/org/apache/sysml/udf/Matrix.java | 28 ++--
.../org/apache/sysml/udf/PackageFunction.java | 39 ++----
.../sysml/udf/PackageRuntimeException.java | 45 -------
.../apache/sysml/udf/lib/BinningWrapper.java | 5 +-
.../org/apache/sysml/udf/lib/DeNaNWrapper.java | 6 +-
.../sysml/udf/lib/DeNegInfinityWrapper.java | 6 +-
.../sysml/udf/lib/DynamicProjectMatrixCP.java | 3 +-
.../sysml/udf/lib/DynamicReadMatrixCP.java | 3 +-
.../sysml/udf/lib/DynamicReadMatrixRcCP.java | 3 +-
.../sysml/udf/lib/DynamicWriteMatrixCP.java | 4 +-
.../org/apache/sysml/udf/lib/GatherWrapper.java | 5 +-
.../org/apache/sysml/udf/lib/OrderWrapper.java | 6 +-
.../sysml/udf/lib/PermutationMatrixWrapper.java | 11 +-
.../org/apache/sysml/udf/lib/TimeWrapper.java | 15 +--
30 files changed, 260 insertions(+), 559 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/67e0d9d2/src/main/java/org/apache/sysml/api/ml/LogisticRegression.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/ml/LogisticRegression.java b/src/main/java/org/apache/sysml/api/ml/LogisticRegression.java
index 4baf17e..5c643ca 100644
--- a/src/main/java/org/apache/sysml/api/ml/LogisticRegression.java
+++ b/src/main/java/org/apache/sysml/api/ml/LogisticRegression.java
@@ -20,7 +20,6 @@
package org.apache.sysml.api.ml;
import java.io.File;
-import java.io.IOException;
import java.util.HashMap;
import org.apache.spark.SparkContext;
@@ -37,12 +36,10 @@ import org.apache.spark.mllib.linalg.Vector;
import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.SQLContext;
-import org.apache.sysml.api.DMLException;
import org.apache.sysml.api.MLContext;
import org.apache.sysml.api.MLOutput;
import org.apache.sysml.api.ml.LogisticRegressionModel;
import org.apache.sysml.api.ml.functions.ConvertSingleColumnToString;
-import org.apache.sysml.parser.ParseException;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.instructions.spark.utils.RDDConverterUtilsExt;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
@@ -470,14 +467,8 @@ public class LogisticRegression extends ProbabilisticClassifier<Vector, Logistic
JavaPairRDD<MatrixIndexes, MatrixBlock> b_out = out.getBinaryBlockedRDD("B_out");
MatrixCharacteristics b_outMC = out.getMatrixCharacteristics("B_out");
return new LogisticRegressionModel(b_out, b_outMC, sc).setParent(this);
- } catch (IOException e) {
+ } catch (Exception e) {
throw new RuntimeException(e);
- } catch (DMLRuntimeException e) {
- throw new RuntimeException(e);
- } catch (DMLException e) {
- throw new RuntimeException(e);
- } catch (ParseException e) {
- throw new RuntimeException(e);
- }
+ }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/67e0d9d2/src/main/java/org/apache/sysml/api/ml/LogisticRegressionModel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/ml/LogisticRegressionModel.java b/src/main/java/org/apache/sysml/api/ml/LogisticRegressionModel.java
index 55aefbb..a16e250 100644
--- a/src/main/java/org/apache/sysml/api/ml/LogisticRegressionModel.java
+++ b/src/main/java/org/apache/sysml/api/ml/LogisticRegressionModel.java
@@ -20,7 +20,6 @@
package org.apache.sysml.api.ml;
import java.io.File;
-import java.io.IOException;
import java.util.HashMap;
import org.apache.spark.SparkContext;
@@ -35,10 +34,8 @@ import org.apache.spark.sql.Row;
import org.apache.spark.sql.RowFactory;
import org.apache.spark.sql.SQLContext;
-import org.apache.sysml.api.DMLException;
import org.apache.sysml.api.MLContext;
import org.apache.sysml.api.MLOutput;
-import org.apache.sysml.parser.ParseException;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.instructions.spark.utils.RDDConverterUtilsExt;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
@@ -175,13 +172,7 @@ public class LogisticRegressionModel extends ProbabilisticClassificationModel<Ve
predictionsNProb = predictionsNProb.join(rawPred, predictionsNProb.col("ID").equalTo(rawPred.col("ID2"))).select("ID", "probability", "prediction", "rawPrediction");
DataFrame dataset1 = RDDConverterUtilsExt.addIDToDataFrame(dataset, sqlContext, "ID");
return dataset1.join(predictionsNProb, dataset1.col("ID").equalTo(predictionsNProb.col("ID"))).orderBy("id");
- } catch (IOException e) {
- throw new RuntimeException(e);
- } catch (DMLRuntimeException e) {
- throw new RuntimeException(e);
- } catch (DMLException e) {
- throw new RuntimeException(e);
- } catch (ParseException e) {
+ } catch (Exception e) {
throw new RuntimeException(e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/67e0d9d2/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java b/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
index dcee148..3cef75b 100644
--- a/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
+++ b/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
@@ -114,6 +114,7 @@ import org.apache.sysml.udf.lib.OrderWrapper;
*
*
*/
+@SuppressWarnings("deprecation")
public class InterProceduralAnalysis
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/67e0d9d2/src/main/java/org/apache/sysml/parser/DMLParseException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/DMLParseException.java b/src/main/java/org/apache/sysml/parser/DMLParseException.java
deleted file mode 100644
index f00e725..0000000
--- a/src/main/java/org/apache/sysml/parser/DMLParseException.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-package org.apache.sysml.parser;
-
-import java.util.ArrayList;
-
-public class DMLParseException extends ParseException
-{
-
-
- /**
- * The version identifier for this Serializable class.
- * Increment only if the <i>serialized</i> form of the
- * class changes.
- */
- private static final long serialVersionUID = 1L;
-
- private String _filename;
-
- private ArrayList<DMLParseException> _exceptionList;
-
- public ArrayList<DMLParseException> getExceptionList(){
- return _exceptionList;
- }
-
- public DMLParseException(String fname){
- super();
- _filename = fname;
- _exceptionList = new ArrayList<DMLParseException>();
- }
-
- public DMLParseException(String fname, String msg){
- super(msg);
- _filename = fname;
- _exceptionList = new ArrayList<DMLParseException>();
- _exceptionList.add(this);
- }
-
- public DMLParseException(String fname, Exception e){
- super();
- _filename = fname;
- _exceptionList = new ArrayList<DMLParseException>();
- String newMsg = e.getMessage();
- if (e instanceof ParseException && !(e instanceof DMLParseException)){
- ParseException parseEx = (ParseException)e;
- int beginLine = -1, beginColumn = -1;
- String errorToken = null;
- if (parseEx.currentToken != null){
- beginLine = parseEx.currentToken.beginLine;
- beginColumn = parseEx.currentToken.beginColumn;
- errorToken = parseEx.currentToken.image;
- newMsg = "ERROR: " + _filename + " -- line " + beginLine + ", column " + beginColumn + " -- " + "Parsing error around token \"" + errorToken + "\"";
- } else {
- newMsg = "ERROR: " + _filename + " -- line " + beginLine + ", column " + beginColumn + " -- " + "Parsing error with unspecified token";
- }
- }
-
- _exceptionList.add(new DMLParseException(_filename, newMsg));
- }
-
- public int size(){
- return _exceptionList.size();
- }
-
- public void add(Exception e){
- if (e instanceof DMLParseException)
- _exceptionList.addAll(((DMLParseException)e).getExceptionList());
- else
- _exceptionList.add(new DMLParseException(this._filename, e));
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/67e0d9d2/src/main/java/org/apache/sysml/parser/DataExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/DataExpression.java b/src/main/java/org/apache/sysml/parser/DataExpression.java
index 84e7c43..562e2ce 100644
--- a/src/main/java/org/apache/sysml/parser/DataExpression.java
+++ b/src/main/java/org/apache/sysml/parser/DataExpression.java
@@ -131,8 +131,8 @@ public class DataExpression extends DataIdentifier
}
public static DataExpression getDataExpression(String functionName, ArrayList<ParameterExpression> passedParamExprs,
- String filename, int blp, int bcp, int elp, int ecp) throws DMLParseException {
-
+ String filename, int blp, int bcp, int elp, int ecp) throws LanguageException
+ {
if (functionName == null || passedParamExprs == null)
return null;
@@ -157,14 +157,13 @@ public class DataExpression extends DataIdentifier
// validate the filename is the first parameter
if (passedParamExprs.size() < 1){
- // throw exception -- must be filename as first parameter
- throw new DMLParseException(dataExpr.getFilename(), dataExpr.printErrorLocation(blp, bcp) + "read method must have at least filename parameter");
+ dataExpr.raiseValidateError("read method must have at least filename parameter", false);
}
ParameterExpression pexpr = (passedParamExprs.size() == 0) ? null : passedParamExprs.get(0);
if ( (pexpr != null) && (!(pexpr.getName() == null) || (pexpr.getName() != null && pexpr.getName().equalsIgnoreCase(DataExpression.IO_FILENAME)))){
- throw new DMLParseException(dataExpr.getFilename(), dataExpr.printErrorLocation(blp, bcp) + "first parameter to read statement must be filename");
+ dataExpr.raiseValidateError("first parameter to read statement must be filename");
} else if( pexpr != null ){
dataExpr.addVarParam(DataExpression.IO_FILENAME, pexpr.getExpr());
}
@@ -175,7 +174,7 @@ public class DataExpression extends DataIdentifier
Expression currExpr = passedParamExprs.get(i).getExpr();
if (dataExpr.getVarParam(currName) != null){
- throw new DMLParseException(dataExpr.getFilename(), currExpr.printErrorLocation() + "attempted to add IOStatement parameter " + currName + " more than once");
+ dataExpr.raiseValidateError("attempted to add IOStatement parameter " + currName + " more than once");
}
// verify parameter names for read function
boolean isValidName = false;
@@ -184,7 +183,7 @@ public class DataExpression extends DataIdentifier
isValidName = true;
}
if (!isValidName){
- throw new DMLParseException(dataExpr.getFilename(), currExpr.printErrorLocation() + "attempted to add invalid read statement parameter " + currName);
+ dataExpr.raiseValidateError("attempted to add invalid read statement parameter " + currName);
}
dataExpr.addVarParam(currName, currExpr);
}
@@ -200,9 +199,7 @@ public class DataExpression extends DataIdentifier
String pname = currExpr.getName();
Expression pexpr = currExpr.getExpr();
if (pname == null){
- throw new DMLParseException(dataExpr.getFilename(), dataExpr.printErrorLocation(blp, bcp) + "for Rand Statment all arguments must be named parameters");
- //LOG.error(dataExpr.printErrorLocation(beginLine, beginColumn) + "for Rand Statment all arguments must be named parameters");
- //throw new ParseException(dataExpr.printErrorLocation(beginLine, beginColumn) + "for Rand Statment all arguments must be named parameters");
+ dataExpr.raiseValidateError("for Rand Statment all arguments must be named parameters");
}
dataExpr.addRandExprParam(pname, pexpr);
}
@@ -224,16 +221,16 @@ public class DataExpression extends DataIdentifier
// check whether named or unnamed parameters are used
if (passedParamExprs.size() < 3){
- throw new DMLParseException(dataExpr.getFilename(), dataExpr.printErrorLocation(blp, bcp) + "for matrix statement, must specify at least 3 arguments (in order): data, rows, cols");
+ dataExpr.raiseValidateError("for matrix statement, must specify at least 3 arguments (in order): data, rows, cols");
}
if (unnamedParamCount > 1){
if (namedParamCount > 0)
- throw new DMLParseException(dataExpr.getFilename(), dataExpr.printErrorLocation(blp, bcp) + "for matrix statement, cannot mix named and unnamed parameters");
+ dataExpr.raiseValidateError("for matrix statement, cannot mix named and unnamed parameters");
if (unnamedParamCount < 3)
- throw new DMLParseException(dataExpr.getFilename(), dataExpr.printErrorLocation(blp, bcp) + "for matrix statement, must specify at least 3 arguments (in order): data, rows, cols");
+ dataExpr.raiseValidateError("for matrix statement, must specify at least 3 arguments (in order): data, rows, cols");
// assume: data, rows, cols, [byRow], [dimNames]
@@ -248,7 +245,7 @@ public class DataExpression extends DataIdentifier
dataExpr.addMatrixExprParam(DataExpression.RAND_DIMNAMES,passedParamExprs.get(4).getExpr());
if (unnamedParamCount > 5)
- throw new DMLParseException(dataExpr.getFilename(), dataExpr.printErrorLocation(blp, bcp) + "for matrix statement, at most 5 arguments supported (in order): data, rows, cols, byrow, dimname");
+ dataExpr.raiseValidateError("for matrix statement, at most 5 arguments supported (in order): data, rows, cols, byrow, dimname");
} else {
@@ -256,7 +253,7 @@ public class DataExpression extends DataIdentifier
ParameterExpression firstParam = passedParamExprs.get(0);
if (firstParam.getName() != null && !firstParam.getName().equals(DataExpression.RAND_DATA)){
// throw exception -- must be filename as first parameter
- throw new DMLParseException(dataExpr.getFilename(), dataExpr.printErrorLocation(blp, bcp) + "matrix method must have data parameter as first parameter or unnamed parameter");
+ dataExpr.raiseValidateError("matrix method must have data parameter as first parameter or unnamed parameter");
} else {
dataExpr.addMatrixExprParam(DataExpression.RAND_DATA, passedParamExprs.get(0).getExpr());
}
@@ -264,7 +261,7 @@ public class DataExpression extends DataIdentifier
for (int i=1; i<passedParamExprs.size(); i++){
if (passedParamExprs.get(i).getName() == null){
// throw exception -- cannot mix named and unnamed parameters
- throw new DMLParseException(dataExpr.getFilename(), dataExpr.printErrorLocation(blp, bcp) + "for matrix statement, cannot mix named and unnamed parameters, only data parameter can be unnammed");
+ dataExpr.raiseValidateError("for matrix statement, cannot mix named and unnamed parameters, only data parameter can be unnammed");
} else {
dataExpr.addMatrixExprParam(passedParamExprs.get(i).getName(), passedParamExprs.get(i).getExpr());
}
@@ -280,7 +277,8 @@ public class DataExpression extends DataIdentifier
} // end method getBuiltinFunctionExpression
- public void addRandExprParam(String paramName, Expression paramValue) throws DMLParseException
+ public void addRandExprParam(String paramName, Expression paramValue)
+ throws LanguageException
{
// check name is valid
boolean found = false;
@@ -293,15 +291,14 @@ public class DataExpression extends DataIdentifier
}
}
if (!found){
-
- throw new DMLParseException(paramValue.getFilename(), paramValue.printErrorLocation() + "unexpected parameter \"" + paramName +
+ raiseValidateError("unexpected parameter \"" + paramName +
"\". Legal parameters for Rand statement are "
+ "(capitalization-sensitive): " + RAND_ROWS
+ ", " + RAND_COLS + ", " + RAND_MIN + ", " + RAND_MAX
+ ", " + RAND_SPARSITY + ", " + RAND_SEED + ", " + RAND_PDF + ", " + RAND_LAMBDA);
}
if (getVarParam(paramName) != null){
- throw new DMLParseException(paramValue.getFilename(), paramValue.printErrorLocation() + "attempted to add Rand statement parameter " + paramValue + " more than once");
+ raiseValidateError("attempted to add Rand statement parameter " + paramValue + " more than once");
}
// Process the case where user provides double values to rows or cols
if (paramName.equals(RAND_ROWS) && paramValue instanceof DoubleIdentifier){
@@ -321,7 +318,8 @@ public class DataExpression extends DataIdentifier
}
- public void addMatrixExprParam(String paramName, Expression paramValue) throws DMLParseException
+ public void addMatrixExprParam(String paramName, Expression paramValue)
+ throws LanguageException
{
// check name is valid
boolean found = false;
@@ -334,14 +332,13 @@ public class DataExpression extends DataIdentifier
}
if (!found){
-
- throw new DMLParseException(paramValue.getFilename(), paramValue.printErrorLocation() + "unexpected parameter \"" + paramName +
+ raiseValidateError("unexpected parameter \"" + paramName +
"\". Legal parameters for matrix statement are "
+ "(capitalization-sensitive): " + RAND_DATA + ", " + RAND_ROWS
+ ", " + RAND_COLS + ", " + RAND_BY_ROW);
}
- if (getVarParam(paramName) != null){
- throw new DMLParseException(paramValue.getFilename(), paramValue.printErrorLocation() + "attempted to add matrix statement parameter " + paramValue + " more than once");
+ if (getVarParam(paramName) != null) {
+ raiseValidateError("attempted to add matrix statement parameter " + paramValue + " more than once");
}
// Process the case where user provides double values to rows or cols
if (paramName.equals(RAND_ROWS) && paramValue instanceof DoubleIdentifier){
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/67e0d9d2/src/main/java/org/apache/sysml/parser/Expression.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/Expression.java b/src/main/java/org/apache/sysml/parser/Expression.java
index 654e7f1..2351901 100644
--- a/src/main/java/org/apache/sysml/parser/Expression.java
+++ b/src/main/java/org/apache/sysml/parser/Expression.java
@@ -30,7 +30,6 @@ import org.apache.sysml.hops.Hop.FileFormatTypes;
public abstract class Expression
{
-
public enum Kind {
UnaryOp, BinaryOp, BooleanOp, BuiltinFunctionOp, ParameterizedBuiltinFunctionOp, DataOp, Data, Literal, RelationalOp, ExtBuiltinFunctionOp, FunctionCallOp
};
@@ -387,15 +386,12 @@ public abstract class Expression
///////////////////////////////////////////////////////////////
// validate error handling (consistent for all expressions)
- /**
- *
- * @param msg
- * @param conditional
- * @throws LanguageException
- */
- public void raiseValidateError( String msg, boolean conditional )
- throws LanguageException
- {
+
+ public void raiseValidateError( String msg ) throws LanguageException {
+ raiseValidateError(msg, false, null);
+ }
+
+ public void raiseValidateError( String msg, boolean conditional ) throws LanguageException {
raiseValidateError(msg, conditional, null);
}
@@ -419,7 +415,7 @@ public abstract class Expression
{
String fullMsg = this.printErrorLocation() + msg;
- LOG.error( fullMsg );
+ //LOG.error( fullMsg ); //no redundant error
if( errorCode != null )
throw new LanguageException( fullMsg, errorCode );
else
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/67e0d9d2/src/main/java/org/apache/sysml/parser/IndexedIdentifier.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/IndexedIdentifier.java b/src/main/java/org/apache/sysml/parser/IndexedIdentifier.java
index a143f56..db0bd18 100644
--- a/src/main/java/org/apache/sysml/parser/IndexedIdentifier.java
+++ b/src/main/java/org/apache/sysml/parser/IndexedIdentifier.java
@@ -591,10 +591,10 @@ public class IndexedIdentifier extends DataIdentifier
return newIndexedIdentifier;
}
- public void setIndices(ArrayList<ArrayList<Expression>> passed) throws DMLParseException {
- if (passed.size() != 2){
- throw new DMLParseException(this.getFilename(), this.printErrorLocation() + "matrix indices must be specified for 2 dimensions");
- }
+ public void setIndices(ArrayList<ArrayList<Expression>> passed) throws LanguageException {
+ if (passed.size() != 2)
+ raiseValidateError("matrix indices must be specified for 2 dimensions");
+
ArrayList<Expression> rowIndices = passed.get(0);
ArrayList<Expression> colIndices = passed.get(1);
@@ -610,7 +610,7 @@ public class IndexedIdentifier extends DataIdentifier
_rowLowerEqualsUpper = true;
}
else {
- throw new DMLParseException(this.getFilename(), this.printErrorLocation() + "row indices are length " + rowIndices.size() + " -- should be either 1 or 2");
+ raiseValidateError("row indices are length " + rowIndices.size() + " -- should be either 1 or 2");
}
// case: both upper and lower are defined
@@ -625,12 +625,12 @@ public class IndexedIdentifier extends DataIdentifier
_colLowerEqualsUpper = true;
}
else {
- throw new DMLParseException(this.getFilename(), this.printErrorLocation() + "column indices are length " + + colIndices.size() + " -- should be either 1 or 2");
+ raiseValidateError("column indices are length " + + colIndices.size() + " -- should be either 1 or 2");
}
if (_rowLowerBound instanceof FunctionCallIdentifier || _rowUpperBound instanceof FunctionCallIdentifier
|| _colLowerBound instanceof FunctionCallIdentifier || _colUpperBound instanceof FunctionCallIdentifier){
- throw new DMLParseException(this.getFilename(), this.printErrorLocation() + "UDF functions not supported for row or column indices");
+ raiseValidateError("UDF functions not supported for row or column indices");
}
}
@@ -648,50 +648,50 @@ public class IndexedIdentifier extends DataIdentifier
@Override
public String toString() {
String retVal = getName();
- if (_rowLowerBound != null || _rowUpperBound != null || _colLowerBound != null || _colUpperBound != null){
- retVal += "[";
-
- if (_rowLowerBound != null && _rowUpperBound != null){
- if (_rowLowerBound.toString().equals(_rowUpperBound.toString()))
+ if (_rowLowerBound != null || _rowUpperBound != null || _colLowerBound != null || _colUpperBound != null) {
+ retVal += "[";
+
+ if (_rowLowerBound != null && _rowUpperBound != null){
+ if (_rowLowerBound.toString().equals(_rowUpperBound.toString()))
+ retVal += _rowLowerBound.toString();
+ else
+ retVal += _rowLowerBound.toString() + ":" + _rowUpperBound.toString();
+ }
+ else {
+ if (_rowLowerBound != null || _rowUpperBound != null){
+ if (_rowLowerBound != null)
retVal += _rowLowerBound.toString();
- else
- retVal += _rowLowerBound.toString() + ":" + _rowUpperBound.toString();
- }
- else {
- if (_rowLowerBound != null || _rowUpperBound != null){
- if (_rowLowerBound != null)
- retVal += _rowLowerBound.toString();
-
- retVal += ":";
-
- if (_rowUpperBound != null)
- retVal += _rowUpperBound.toString();
- }
- }
- retVal += ",";
+ retVal += ":";
+
+ if (_rowUpperBound != null)
+ retVal += _rowUpperBound.toString();
+ }
+ }
- if (_colLowerBound != null && _colUpperBound != null){
- if (_colLowerBound.toString().equals(_colUpperBound.toString()))
+ retVal += ",";
+
+ if (_colLowerBound != null && _colUpperBound != null){
+ if (_colLowerBound.toString().equals(_colUpperBound.toString()))
+ retVal += _colLowerBound.toString();
+ else
+ retVal += _colLowerBound.toString() + ":" + _colUpperBound.toString();
+ }
+ else {
+ if (_colLowerBound != null || _colUpperBound != null) {
+
+ if (_colLowerBound != null)
retVal += _colLowerBound.toString();
- else
- retVal += _colLowerBound.toString() + ":" + _colUpperBound.toString();
- }
- else {
- if (_colLowerBound != null || _colUpperBound != null) {
-
- if (_colLowerBound != null)
- retVal += _colLowerBound.toString();
-
- retVal += ":";
-
- if (_colUpperBound != null)
- retVal += _colUpperBound.toString();
- }
+
+ retVal += ":";
+
+ if (_colUpperBound != null)
+ retVal += _colUpperBound.toString();
}
-
-
- retVal += "]";
+ }
+
+
+ retVal += "]";
}
return retVal;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/67e0d9d2/src/main/java/org/apache/sysml/parser/OutputStatement.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/OutputStatement.java b/src/main/java/org/apache/sysml/parser/OutputStatement.java
index cb3cb3a..5cb66e1 100644
--- a/src/main/java/org/apache/sysml/parser/OutputStatement.java
+++ b/src/main/java/org/apache/sysml/parser/OutputStatement.java
@@ -70,42 +70,36 @@ public class OutputStatement extends Statement
* @param ecp
* @throws DMLParseException
*/
- OutputStatement(String fname, FunctionCallIdentifier fci,
+ public OutputStatement(String fname, FunctionCallIdentifier fci,
String filename, int blp, int bcp, int elp, int ecp)
- throws DMLParseException
- {
-
- this.setAllPositions(filename, blp, bcp, elp, ecp);
+ throws LanguageException
+ {
+ setAllPositions(filename, blp, bcp, elp, ecp);
DataOp op = Expression.DataOp.WRITE;
ArrayList<ParameterExpression> passedExprs = fci.getParamExprs();
_paramsExpr = new DataExpression(op, new HashMap<String,Expression>(),
filename, blp, bcp, elp, ecp);
- DMLParseException runningList = new DMLParseException(fname);
//check number parameters and proceed only if this will not cause errors
if (passedExprs.size() < 2)
- runningList.add(new DMLParseException(fci.getFilename(), fci.printErrorLocation() + "write method must specify both variable to write to file, and filename to write variable to"));
+ raiseValidateError("write method must specify both variable to write to file, and filename to write variable to");
else
{
ParameterExpression firstParam = passedExprs.get(0);
if (firstParam.getName() != null || (!(firstParam.getExpr() instanceof DataIdentifier)))
- runningList.add(new DMLParseException(fci.getFilename(), fci.printErrorLocation() + "first argument to write method must be name of variable to be written out"));
+ raiseValidateError("first argument to write method must be name of variable to be written out");
else
_id = (DataIdentifier)firstParam.getExpr();
ParameterExpression secondParam = passedExprs.get(1);
if (secondParam.getName() != null || (secondParam.getName() != null && secondParam.getName().equals(DataExpression.IO_FILENAME)))
- runningList.add(new DMLParseException(fci.getFilename(), fci.printErrorLocation() + "second argument to write method must be filename of file variable written to"));
+ raiseValidateError("second argument to write method must be filename of file variable written to");
else
addExprParam(DataExpression.IO_FILENAME, secondParam.getExpr(), false);
for (int i = 2; i< passedExprs.size(); i++){
ParameterExpression currParam = passedExprs.get(i);
- try {
- addExprParam(currParam.getName(), currParam.getExpr(), false);
- } catch (DMLParseException e){
- runningList.add(e);
- }
+ addExprParam(currParam.getName(), currParam.getExpr(), false);
}
if (fname.equals("writeMM")){
StringIdentifier writeMMExpr = new StringIdentifier(DataExpression.FORMAT_TYPE_VALUE_MATRIXMARKET,
@@ -118,9 +112,6 @@ public class OutputStatement extends Statement
addExprParam(DataExpression.FORMAT_TYPE, delimitedExpr, false);
}
}
-
- if (runningList.size() > 0)
- throw runningList;
}
public void setExprParam(String name, Expression value) {
@@ -134,20 +125,16 @@ public class OutputStatement extends Statement
return false;
}
- public void addExprParam(String name, Expression value, boolean fromMTDFile) throws DMLParseException
+ public void addExprParam(String name, Expression value, boolean fromMTDFile)
+ throws LanguageException
{
- DMLParseException runningList = new DMLParseException(value.getFilename());
-
if( _paramsExpr.getVarParam(name) != null )
- runningList.add(new DMLParseException(value.getFilename(), value.printErrorLocation() + "attempted to add IOStatement parameter " + name + " more than once"));
+ raiseValidateError("attempted to add IOStatement parameter " + name + " more than once", false);
if( !OutputStatement.isValidParamName(name) )
- runningList.add(new DMLParseException(value.getFilename(), value.printErrorLocation() + "attempted to add invalid write statement parameter: " + name));
+ raiseValidateError("attempted to add invalid write statement parameter: " + name, false);
_paramsExpr.addVarParam(name, value);
-
- if (runningList.size() > 0)
- throw runningList;
}
// rewrites statement to support function inlining (create deep copy)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/67e0d9d2/src/main/java/org/apache/sysml/parser/ParseException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/ParseException.java b/src/main/java/org/apache/sysml/parser/ParseException.java
index 5bd9a2b..d4338bf 100644
--- a/src/main/java/org/apache/sysml/parser/ParseException.java
+++ b/src/main/java/org/apache/sysml/parser/ParseException.java
@@ -21,46 +21,49 @@ package org.apache.sysml.parser;
import java.util.List;
+import org.apache.sysml.api.DMLException;
import org.apache.sysml.parser.common.CustomErrorListener.ParseIssue;
/**
* This exception is thrown when parse issues are encountered.
*
- * NOTE: MB: Originally, this exception was generated with javacc.
- * Unfortunately, many classes directly use this exception. Hence, on removing
- * javacc we kept a condensed version of it. TODO: eventually we should remove
- * the token class and create our own exception or just use LanguageException.
*/
-public class ParseException extends Exception {
+public class ParseException extends DMLException
+{
+ private static final long serialVersionUID = 9199966053655385928L;
/**
- * The version identifier for this Serializable class. Increment only if the
- * <i>serialized</i> form of the class changes.
+ * List of issues that happened during parsing. Typically set by the error
+ * listener.
+ */
+ private List<ParseIssue> _parseIssues;
+
+ /**
+ * The DML/PyDML script string. Used to display the original lines where the
+ * parse issues occurred.
*/
- private static final long serialVersionUID = 1L;
+ private String _scriptString;
/**
- * The following constructors are for use by you for whatever purpose you
- * can think of. Constructing the exception in this manner makes the
- * exception behave in the normal way - i.e., as documented in the class
- * "Throwable". The fields "errorToken", "expectedTokenSequences", and
- * "tokenImage" do not contain relevant information. The JavaCC generated
- * code does not use these constructors.
+ * Standard exception message. If no list of parse issues exists, this
+ * message can be used to display a parse exception message that doesn't
+ * relate to the list of parse issues.
*/
+ private String _message;
+
public ParseException() {
super();
}
- /** Constructor with message. */
public ParseException(String message) {
super(message);
- this.message = message;
+ _message = message;
}
public ParseException(String message, Exception e) {
super(message, e);
- this.message = message;
+ _message = message;
}
/**
@@ -75,57 +78,17 @@ public class ParseException extends Exception {
*/
public ParseException(List<ParseIssue> parseIssues, String scriptString) {
super();
- this.parseIssues = parseIssues;
- this.scriptString = scriptString;
+ _parseIssues = parseIssues;
+ _scriptString = scriptString;
}
/**
- * This is the last token that has been consumed successfully. If this
- * object has been created due to a parse issue, the token following this
- * token will (therefore) be the first error token.
- */
- public Token currentToken;
-
- /**
- * Each entry in this array is an array of integers. Each array of integers
- * represents a sequence of tokens (by their ordinal values) that is
- * expected at this point of the parse.
- */
- public int[][] expectedTokenSequences;
-
- /**
- * This is a reference to the "tokenImage" array of the generated parser
- * within which the parse issue occurred. This array is defined in the
- * generated ...Constants interface.
- */
- public String[] tokenImage;
-
- /**
- * List of issues that happened during parsing. Typically set by the error
- * listener.
- */
- private List<ParseIssue> parseIssues;
-
- /**
- * The DML/PyDML script string. Used to display the original lines where the
- * parse issues occurred.
- */
- private String scriptString;
-
- /**
- * Standard exception message. If no list of parse issues exists, this
- * message can be used to display a parse exception message that doesn't
- * relate to the list of parse issues.
- */
- private String message;
-
- /**
* Obtain the list of parse issues that occurred.
*
* @return the list of parse issues
*/
public List<ParseIssue> getParseIssues() {
- return parseIssues;
+ return _parseIssues;
}
/**
@@ -135,7 +98,7 @@ public class ParseException extends Exception {
* the list of parse issues
*/
public void setParseIssues(List<ParseIssue> parseIssues) {
- this.parseIssues = parseIssues;
+ _parseIssues = parseIssues;
}
/**
@@ -144,7 +107,7 @@ public class ParseException extends Exception {
* @return the original DML/PyDML script string
*/
public String getScriptString() {
- return scriptString;
+ return _scriptString;
}
/**
@@ -154,7 +117,7 @@ public class ParseException extends Exception {
* the original DML/PyDML script string
*/
public void setScriptString(String scriptString) {
- this.scriptString = scriptString;
+ _scriptString = scriptString;
}
/**
@@ -164,11 +127,7 @@ public class ParseException extends Exception {
* greater than 0, <code>false</code> otherwise
*/
public boolean hasParseIssues() {
- if ((parseIssues != null) && (parseIssues.size() > 0)) {
- return true;
- } else {
- return false;
- }
+ return (_parseIssues != null && _parseIssues.size() > 0);
}
/**
@@ -178,15 +137,8 @@ public class ParseException extends Exception {
*/
@Override
public String getMessage() {
- if (!hasParseIssues()) {
- if (message != null) {
- return message;
- } else {
- return "No parse issue message.";
- }
- } else {
- return generateParseIssuesMessage();
- }
+ return hasParseIssues() ? generateParseIssuesMessage() :
+ (_message != null) ? _message : "No parse issue message.";
}
/**
@@ -196,20 +148,19 @@ public class ParseException extends Exception {
* @return String representing the list of parse issues.
*/
private String generateParseIssuesMessage() {
- if (scriptString == null) {
+ if (_scriptString == null)
return "No script string available.";
- }
- String[] scriptLines = scriptString.split("\\n");
+
+ String[] scriptLines = _scriptString.split("\\n");
StringBuilder sb = new StringBuilder();
sb.append("\n--------------------------------------------------------------");
- if (parseIssues.size() == 1) {
+ if (_parseIssues.size() == 1)
sb.append("\nThe following parse issue was encountered:\n");
- } else {
- sb.append("\nThe following " + parseIssues.size() + " parse issues were encountered:\n");
- }
+ else
+ sb.append("\nThe following " + _parseIssues.size() + " parse issues were encountered:\n");
int count = 1;
- for (ParseIssue parseIssue : parseIssues) {
- if (parseIssues.size() > 1) {
+ for (ParseIssue parseIssue : _parseIssues) {
+ if (_parseIssues.size() > 1) {
sb.append("#");
sb.append(count++);
sb.append(" ");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/67e0d9d2/src/main/java/org/apache/sysml/parser/Statement.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/Statement.java b/src/main/java/org/apache/sysml/parser/Statement.java
index bb35ecf..532dbfa 100644
--- a/src/main/java/org/apache/sysml/parser/Statement.java
+++ b/src/main/java/org/apache/sysml/parser/Statement.java
@@ -132,5 +132,39 @@ public abstract class Statement
return "INFO: " + _filename + " -- line " + beginLine + ", column " + beginColumn + " -- ";
}
+ public void raiseValidateError( String msg ) throws LanguageException {
+ raiseValidateError(msg, false, null);
+ }
+
+ public void raiseValidateError( String msg, boolean conditional ) throws LanguageException {
+ raiseValidateError(msg, conditional, null);
+ }
+ /**
+ *
+ * @param msg
+ * @param conditional
+ * @param code
+ * @throws LanguageException
+ */
+ public void raiseValidateError( String msg, boolean conditional, String errorCode )
+ throws LanguageException
+ {
+ if( conditional ) //warning if conditional
+ {
+ String fullMsg = this.printWarningLocation() + msg;
+
+ LOG.warn( fullMsg );
+ }
+ else //error and exception if unconditional
+ {
+ String fullMsg = this.printErrorLocation() + msg;
+
+ //LOG.error( fullMsg ); //no redundant error
+ if( errorCode != null )
+ throw new LanguageException( fullMsg, errorCode );
+ else
+ throw new LanguageException( fullMsg );
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/67e0d9d2/src/main/java/org/apache/sysml/parser/StatementBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/StatementBlock.java b/src/main/java/org/apache/sysml/parser/StatementBlock.java
index c6290bb..b3f0cec 100644
--- a/src/main/java/org/apache/sysml/parser/StatementBlock.java
+++ b/src/main/java/org/apache/sysml/parser/StatementBlock.java
@@ -198,7 +198,6 @@ public class StatementBlock extends LiveVariableAnalysis
FunctionCallIdentifier fcall = (FunctionCallIdentifier) sourceExpr;
FunctionStatementBlock fblock = dmlProg.getFunctionStatementBlock(fcall.getNamespace(), fcall.getName());
if (fblock == null){
- LOG.error(sourceExpr.printErrorLocation() + "function " + fcall.getName() + " is undefined in namespace " + fcall.getNamespace());
throw new LanguageException(sourceExpr.printErrorLocation() + "function " + fcall.getName() + " is undefined in namespace " + fcall.getNamespace());
}
if( !rIsInlineableFunction(fblock, dmlProg) )
@@ -224,7 +223,6 @@ public class StatementBlock extends LiveVariableAnalysis
FunctionCallIdentifier fcall = (FunctionCallIdentifier) sourceExpr;
FunctionStatementBlock fblock = dmlProg.getFunctionStatementBlock(fcall.getNamespace(),fcall.getName());
if (fblock == null){
- LOG.error(sourceExpr.printErrorLocation() + "function " + fcall.getName() + " is undefined in namespace " + fcall.getNamespace());
throw new LanguageException(sourceExpr.printErrorLocation() + "function " + fcall.getName() + " is undefined in namespace " + fcall.getNamespace());
}
@@ -415,8 +413,7 @@ public class StatementBlock extends LiveVariableAnalysis
FunctionCallIdentifier fcall = (FunctionCallIdentifier) sourceExpr;
FunctionStatementBlock fblock = dmlProg.getFunctionStatementBlock(fcall.getNamespace(), fcall.getName());
if (fblock == null){
- LOG.error(fcall.printErrorLocation() + "function " + fcall.getName() + " is undefined in namespace " + fcall.getNamespace());
- throw new LanguageException(fcall.printErrorLocation() + "function " + fcall.getName() + " is undefined in namespace " + fcall.getNamespace());
+ fcall.raiseValidateError("function " + fcall.getName() + " is undefined in namespace " + fcall.getNamespace(), false);
}
FunctionStatement fstmt = (FunctionStatement)fblock.getStatement(0);
@@ -430,8 +427,7 @@ public class StatementBlock extends LiveVariableAnalysis
String prefix = _seq.getNextID() + "_";
if (fstmt.getBody().size() > 1){
- LOG.error(sourceExpr.printErrorLocation() + "rewritable function can only have 1 statement block");
- throw new LanguageException(sourceExpr.printErrorLocation() + "rewritable function can only have 1 statement block");
+ sourceExpr.raiseValidateError("rewritable function can only have 1 statement block", false);
}
StatementBlock sblock = fstmt.getBody().get(0);
@@ -452,8 +448,7 @@ public class StatementBlock extends LiveVariableAnalysis
else {
// use default value for parameter
if (fstmt.getInputParams().get(i).getDefaultValue() == null){
- LOG.error(currFormalParam.printErrorLocation() + "default parameter for " + currFormalParam + " is undefined");
- throw new LanguageException(currFormalParam.printErrorLocation() + "default parameter for " + currFormalParam + " is undefined");
+ currFormalParam.raiseValidateError("default parameter for " + currFormalParam + " is undefined", false);
}
currCallParam = new DataIdentifier(fstmt.getInputParams().get(i).getDefaultValue());
currCallParam.setAllPositions( fstmt.getInputParams().get(i).getFilename(),
@@ -497,8 +492,7 @@ public class StatementBlock extends LiveVariableAnalysis
DataIdentifier newTarget = null;
if (current instanceof AssignmentStatement){
if (i > 0) {
- LOG.error(current.printErrorLocation() + "Assignment statement cannot return multiple values");
- throw new LanguageException(current.printErrorLocation() + "Assignment statement cannot return multiple values");
+ fstmt.raiseValidateError("Assignment statement cannot return multiple values", false);
}
newTarget = new DataIdentifier(((AssignmentStatement)current).getTarget());
}
@@ -873,8 +867,7 @@ public class StatementBlock extends LiveVariableAnalysis
VariableSet updated = s.variablesUpdated();
if (s instanceof WhileStatement || s instanceof IfStatement || s instanceof ForStatement){
- LOG.error(s.printErrorLocation() + "control statement (while / for / if) cannot be in generic statement block");
- throw new LanguageException(s.printErrorLocation() + "control statement (while / for / if) cannot be in generic statement block");
+ raiseValidateError("control statement (while / for / if) cannot be in generic statement block", false);
}
if (read != null){
@@ -998,7 +991,7 @@ public class StatementBlock extends LiveVariableAnalysis
{
String fullMsg = this.printErrorLocation() + msg;
- LOG.error( fullMsg );
+ //LOG.error( fullMsg ); //no redundant error
if( errorCode != null )
throw new LanguageException( fullMsg, errorCode );
else
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/67e0d9d2/src/main/java/org/apache/sysml/parser/common/CustomErrorListener.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/common/CustomErrorListener.java b/src/main/java/org/apache/sysml/parser/common/CustomErrorListener.java
index 5d2ebc6..6b62499 100644
--- a/src/main/java/org/apache/sysml/parser/common/CustomErrorListener.java
+++ b/src/main/java/org/apache/sysml/parser/common/CustomErrorListener.java
@@ -67,6 +67,9 @@ public class CustomErrorListener extends BaseErrorListener {
public void validationError(int line, int charPositionInLine, String msg) {
parseIssues
.add(new ParseIssue(line, charPositionInLine, msg, currentFileName, ParseIssueType.VALIDATION_ERROR));
+
+ //TODO MB: we should not redundantly log errors here
+ //(this also applies for the two other methods below).
try {
setAtleastOneError(true);
// Print error messages with file name
@@ -94,6 +97,7 @@ public class CustomErrorListener extends BaseErrorListener {
public void validationWarning(int line, int charPositionInLine, String msg) {
parseIssues.add(new ParseIssue(line, charPositionInLine, msg, currentFileName,
ParseIssueType.VALIDATION_WARNING));
+
try {
// atleastOneError = true; ---> not an error, just warning
// Print error messages with file name
@@ -273,6 +277,8 @@ public class CustomErrorListener extends BaseErrorListener {
* to the user.
*/
public enum ParseIssueType {
+ //TODO MB: This classification is misleading as it only refers to variations of parsing issues.
+ //We need to consolidate the handling of parsing issues and actual validation issues (see validateParseTree).
SYNTAX_ERROR("Syntax error"), VALIDATION_ERROR("Validation error"), VALIDATION_WARNING("Validation warning");
ParseIssueType(String text) {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/67e0d9d2/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
index af1671a..1485e51 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
@@ -57,7 +57,6 @@ import org.apache.sysml.udf.ExternalFunctionInvocationInstruction;
import org.apache.sysml.udf.FunctionParameter;
import org.apache.sysml.udf.Matrix;
import org.apache.sysml.udf.PackageFunction;
-import org.apache.sysml.udf.PackageRuntimeException;
import org.apache.sysml.udf.Scalar;
import org.apache.sysml.udf.FunctionParameter.FunctionParameterType;
import org.apache.sysml.udf.BinaryObject;
@@ -182,7 +181,7 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
}
}
catch (Exception e){
- throw new PackageRuntimeException(this.printBlockErrorLocation() + "Error exporting input variables to HDFS", e);
+ throw new DMLRuntimeException(this.printBlockErrorLocation() + "Error exporting input variables to HDFS", e);
}
// convert block to cell
@@ -194,7 +193,7 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
this.executeInstructions(tempInst,ec);
} catch (Exception e) {
- throw new PackageRuntimeException(this.printBlockErrorLocation() + "Error executing "
+ throw new DMLRuntimeException(this.printBlockErrorLocation() + "Error executing "
+ tempInst.toString(), e);
}
}
@@ -207,7 +206,7 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
executeInstruction(ec, (ExternalFunctionInvocationInstruction) _inst.get(i));
}
catch(Exception e) {
- throw new PackageRuntimeException(this.printBlockErrorLocation() +
+ throw new DMLRuntimeException(this.printBlockErrorLocation() +
"Failed to execute instruction " + _inst.get(i).toString(), e);
}
}
@@ -222,7 +221,7 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
this.executeInstructions(tempInst, ec);
} catch (Exception e) {
- throw new PackageRuntimeException(this.printBlockErrorLocation() + "Failed to execute instruction "
+ throw new DMLRuntimeException(this.printBlockErrorLocation() + "Failed to execute instruction "
+ cell2BlockInst.toString(), e);
}
}
@@ -238,7 +237,6 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
* @param params
* @return
*/
-
protected String getParameterString(ArrayList<DataIdentifier> params) {
String parameterString = "";
@@ -292,7 +290,7 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
// class name cannot be null, however, configFile and execLocation can
// be null
if (className == null)
- throw new PackageRuntimeException(this.printBlockErrorLocation() + ExternalFunctionStatement.CLASS_NAME + " not provided!");
+ throw new RuntimeException(this.printBlockErrorLocation() + ExternalFunctionStatement.CLASS_NAME + " not provided!");
// assemble input and output param strings
String inputParameterString = getParameterString(getInputParams());
@@ -427,7 +425,7 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
//c2binst.add(CPInstructionParser.parseSingleInstruction("CP" + Lops.OPERAND_DELIMITOR + "cpvar"+Lops.OPERAND_DELIMITOR+ outLabels.get(i) + Lops.OPERAND_DELIMITOR + matrices.get(i).getName()));
}
} catch (Exception e) {
- throw new PackageRuntimeException(this.printBlockErrorLocation() + "error generating instructions", e);
+ throw new RuntimeException(this.printBlockErrorLocation() + "error generating instructions", e);
}
//LOGGING instructions
@@ -550,7 +548,7 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
b2cinst.add(rmInst);
}
} catch (Exception e) {
- throw new PackageRuntimeException(e);
+ throw new RuntimeException(e);
}
//LOG instructions
@@ -565,31 +563,14 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
//BEGIN FUNCTION PATCH
if( !matricesNoReblock.isEmpty() )
- {
- //if( b2cinst==null )
- // b2cinst = new ArrayList<Instruction>();
-
+ {
for( int i=0; i<matricesNoReblock.size(); i++ )
{
String scratchSpaceLoc = ConfigurationManager.getScratchSpace();
-
- try{
- String filename = scratchSpaceLoc +
+ String filename = scratchSpaceLoc +
Lop.FILE_SEPARATOR + Lop.PROCESS_PREFIX + DMLScript.getUUID() + Lop.FILE_SEPARATOR +
_otherParams.get(ExternalFunctionStatement.CLASS_NAME) + _runID + "_" + i + "Input";
- //String outLabel = matricesNoReblock.get(i).getName()+"_extFnInput";
- //Instruction createInst = VariableCPInstruction.prepareCreateVariableInstruction(outLabel, filename, false, OutputInfo.outputInfoToString(OutputInfo.TextCellOutputInfo));
- //Instruction cpInst = VariableCPInstruction.prepareCopyInstruction( matricesNoReblock.get(i).getName(), outLabel);
-
- unBlockedFileNames.put(matricesNoReblock.get(i).getName(), filename); //
-
- //b2cinst.add(createInst);
- //b2cinst.add(cpInst);
- }
- catch (Exception e) {
- throw new PackageRuntimeException(e);
- }
-
+ unBlockedFileNames.put(matricesNoReblock.get(i).getName(), filename);
}
}
//END FUNCTION PATCH
@@ -612,7 +593,7 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
String configFile = inst.getConfigFile();
if (className == null)
- throw new PackageRuntimeException(this.printBlockErrorLocation() + "Class name can't be null");
+ throw new DMLRuntimeException(this.printBlockErrorLocation() + "Class name can't be null");
// create instance of package function.
Object o;
@@ -623,11 +604,11 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
}
catch (Exception e)
{
- throw new PackageRuntimeException(this.printBlockErrorLocation() + "Error generating package function object " ,e );
+ throw new DMLRuntimeException(this.printBlockErrorLocation() + "Error generating package function object " ,e );
}
if (!(o instanceof PackageFunction))
- throw new PackageRuntimeException(this.printBlockErrorLocation() + "Class is not of type PackageFunction");
+ throw new DMLRuntimeException(this.printBlockErrorLocation() + "Class is not of type PackageFunction");
PackageFunction func = (PackageFunction) o;
@@ -659,7 +640,7 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
// make sure they are of equal size first
if (outputs.size() != returnFunc.getNumFunctionOutputs()) {
- throw new PackageRuntimeException(
+ throw new DMLRuntimeException(
"Number of function outputs ("+returnFunc.getNumFunctionOutputs()+") " +
"does not match with declaration ("+outputs.size()+").");
}
@@ -677,7 +658,7 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
if (!(tokens.get(0).equals(getFunctionParameterDataTypeString(FunctionParameterType.Matrix)))
|| !(tokens.get(2).equals(getMatrixValueTypeString(m.getValueType())))) {
- throw new PackageRuntimeException(
+ throw new DMLRuntimeException(
"Function output '"+outputs.get(i)+"' does not match with declaration.");
}
@@ -686,11 +667,6 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
MatrixObject newVar = createOutputMatrixObject( m );
newVar.setVarName(varName);
- /* cleanup not required because done at central position (FunctionCallCPInstruction)
- MatrixObjectNew oldVar = (MatrixObjectNew)getVariable(varName);
- if( oldVar!=null )
- oldVar.clearData();*/
-
//getVariables().put(varName, newVar); //put/override in local symbol table
ec.setVariable(varName, newVar);
@@ -703,7 +679,7 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
if (!tokens.get(0).equals(getFunctionParameterDataTypeString(FunctionParameterType.Scalar))
|| !tokens.get(2).equals(
getScalarValueTypeString(s.getScalarType()))) {
- throw new PackageRuntimeException(
+ throw new DMLRuntimeException(
"Function output '"+outputs.get(i)+"' does not match with declaration.");
}
@@ -727,7 +703,7 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
scalarObject = new StringObject(tokens.get(1), s.getValue());
break;
default:
- throw new PackageRuntimeException(
+ throw new DMLRuntimeException(
"Unknown scalar value type '"+type+"' of output '"+outputs.get(i)+"'.");
}
@@ -738,17 +714,17 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
if (returnFunc.getFunctionOutput(i).getType() == FunctionParameterType.Object) {
if (!tokens.get(0).equals(getFunctionParameterDataTypeString(FunctionParameterType.Object))) {
- throw new PackageRuntimeException(
+ throw new DMLRuntimeException(
"Function output '"+outputs.get(i)+"' does not match with declaration.");
}
- throw new PackageRuntimeException(
+ throw new DMLRuntimeException(
"Object types not yet supported");
// continue;
}
- throw new PackageRuntimeException(
+ throw new DMLRuntimeException(
"Unknown data type '"+returnFunc.getFunctionOutput(i).getType()+"' " +
"of output '"+outputs.get(i)+"'.");
}
@@ -768,19 +744,11 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
* @param scalarType
* @return
*/
-
protected String getScalarValueTypeString(ScalarValueType scalarType) {
-
- if (scalarType.equals(ScalarValueType.Double))
- return "Double";
- if (scalarType.equals(ScalarValueType.Integer))
- return "Integer";
- if (scalarType.equals(ScalarValueType.Boolean))
- return "Boolean";
if (scalarType.equals(ScalarValueType.Text))
return "String";
-
- throw new PackageRuntimeException("Unknown scalar value type");
+ else
+ return scalarType.toString();
}
/**
@@ -791,15 +759,12 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
* @param metaData
* @param variableMapping
*/
- protected void setupInputs (PackageFunction func, String inputParams,
- LocalVariableMap variableMapping) {
-
+ protected void setupInputs (PackageFunction func, String inputParams, LocalVariableMap variableMapping) {
ArrayList<String> inputs = getParameters(inputParams);
ArrayList<FunctionParameter> inputObjects = getInputObjects(inputs, variableMapping);
func.setNumFunctionInputs(inputObjects.size());
for (int i = 0; i < inputObjects.size(); i++)
func.setInput(inputObjects.get(i), i);
-
}
/**
@@ -868,17 +833,10 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
* @return
*/
protected ScalarValueType getScalarValueType(String string) {
- if (string.equals("Double"))
- return ScalarValueType.Double;
- if (string.equals("Integer"))
- return ScalarValueType.Integer;
- if (string.equals("Boolean"))
- return ScalarValueType.Boolean;
if (string.equals("String"))
return ScalarValueType.Text;
-
- throw new PackageRuntimeException("Unknown scalar type");
-
+ else
+ return ScalarValueType.valueOf(string);
}
/**
@@ -887,15 +845,8 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
* @param t
* @return
*/
-
protected String getMatrixValueTypeString(Matrix.ValueType t) {
- if (t.equals(Matrix.ValueType.Double))
- return "Double";
-
- if (t.equals(Matrix.ValueType.Integer))
- return "Integer";
-
- throw new PackageRuntimeException("Unknown matrix value type");
+ return t.toString();
}
/**
@@ -904,16 +855,8 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
* @param string
* @return
*/
-
- protected org.apache.sysml.udf.Matrix.ValueType getMatrixValueType(String string) {
-
- if (string.equals("Double"))
- return Matrix.ValueType.Double;
- if (string.equals("Integer"))
- return Matrix.ValueType.Integer;
-
- throw new PackageRuntimeException("Unknown matrix value type");
-
+ protected Matrix.ValueType getMatrixValueType(String string) {
+ return Matrix.ValueType.valueOf(string);
}
/**
@@ -950,8 +893,7 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
if (d.equals(DataType.OBJECT))
return "Object";
- throw new PackageRuntimeException("Should never come here");
-
+ throw new RuntimeException("Should never come here");
}
/**
@@ -961,16 +903,7 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
* @return
*/
protected String getFunctionParameterDataTypeString(FunctionParameterType t) {
- if (t.equals(FunctionParameterType.Matrix))
- return "Matrix";
-
- if (t.equals(FunctionParameterType.Scalar))
- return "Scalar";
-
- if (t.equals(FunctionParameterType.Object))
- return "Object";
-
- throw new PackageRuntimeException("Should never come here");
+ return t.toString();
}
/**
@@ -992,7 +925,7 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
if (v.equals(ValueType.STRING))
return "String";
- throw new PackageRuntimeException("Should never come here");
+ throw new RuntimeException("Should never come here");
}
public void printMe() {
@@ -1002,8 +935,7 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
}
}
- public HashMap<String,String> getOtherParams()
- {
+ public HashMap<String,String> getOtherParams() {
return _otherParams;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/67e0d9d2/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java
index 3740e83..dac0389 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java
@@ -37,7 +37,6 @@ import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
import org.apache.sysml.udf.ExternalFunctionInvocationInstruction;
import org.apache.sysml.udf.Matrix;
-import org.apache.sysml.udf.PackageRuntimeException;
/**
* CP external function program block, that overcomes the need for
@@ -135,7 +134,7 @@ public class ExternalFunctionProgramBlockCP extends ExternalFunctionProgramBlock
// class name cannot be null, however, configFile and execLocation can be null
if (className == null)
- throw new PackageRuntimeException(this.printBlockErrorLocation() + ExternalFunctionStatement.CLASS_NAME + " not provided!");
+ throw new RuntimeException(this.printBlockErrorLocation() + ExternalFunctionStatement.CLASS_NAME + " not provided!");
// assemble input and output param strings
String inputParameterString = getParameterString(getInputParams());
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/67e0d9d2/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestExtFunctCP.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestExtFunctCP.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestExtFunctCP.java
index 2553105..eda6be6 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestExtFunctCP.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestExtFunctCP.java
@@ -26,7 +26,6 @@ import org.apache.sysml.runtime.matrix.data.OutputInfo;
import org.apache.sysml.udf.FunctionParameter;
import org.apache.sysml.udf.Matrix;
import org.apache.sysml.udf.PackageFunction;
-import org.apache.sysml.udf.PackageRuntimeException;
import org.apache.sysml.udf.Matrix.ValueType;
/**
@@ -87,7 +86,7 @@ public class PerfTestExtFunctCP extends PackageFunction
}
catch(Exception e)
{
- throw new PackageRuntimeException("Error executing generic test extfunct.", e);
+ throw new RuntimeException("Error executing generic test extfunct.", e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/67e0d9d2/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/DataConverter.java b/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
index 32d47f2..22a3363 100644
--- a/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
@@ -47,7 +47,6 @@ import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
import org.apache.sysml.runtime.matrix.data.SparseBlock;
-import org.apache.sysml.udf.Matrix;
/**
@@ -842,12 +841,9 @@ public class DataConverter
public static Array2DRowRealMatrix convertToArray2DRowRealMatrix(MatrixObject mo)
throws DMLRuntimeException
{
- Matrix.ValueType vt = (mo.getValueType() == ValueType.DOUBLE ? Matrix.ValueType.Double : Matrix.ValueType.Integer);
- Matrix mathInput = new Matrix(mo.getFileName(), mo.getNumRows(), mo.getNumColumns(), vt);
- mathInput.setMatrixObject(mo);
- double[][] data = mathInput.getMatrixAsDoubleArray();
- Array2DRowRealMatrix matrixInput = new Array2DRowRealMatrix(data, false);
-
- return matrixInput;
+ MatrixBlock mb = mo.acquireRead();
+ double[][] data = DataConverter.convertToDoubleMatrix(mb);
+ mo.release();
+ return new Array2DRowRealMatrix(data, false);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/67e0d9d2/src/main/java/org/apache/sysml/udf/Matrix.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/Matrix.java b/src/main/java/org/apache/sysml/udf/Matrix.java
index 6aa733c..98f1d4b 100644
--- a/src/main/java/org/apache/sysml/udf/Matrix.java
+++ b/src/main/java/org/apache/sysml/udf/Matrix.java
@@ -141,29 +141,23 @@ public class Matrix extends FunctionParameter
* representation.
*
* @return
+ * @throws DMLRuntimeException
+ * @throws IOException
*/
public double[][] getMatrixAsDoubleArray()
+ throws DMLRuntimeException, IOException
{
double[][] ret = null;
- try
- {
- if( _mo != null ) //CP ext function
- {
- MatrixBlock mb = _mo.acquireRead();
- ret = DataConverter.convertToDoubleMatrix( mb );
- _mo.release();
- }
- else //traditional ext function (matrix file produced by reblock)
- {
- MatrixReader reader = MatrixReaderFactory.createMatrixReader(InputInfo.TextCellInputInfo);
- MatrixBlock mb = reader.readMatrixFromHDFS(this.getFilePath(), _rows, _cols, -1, -1, -1);
- ret = DataConverter.convertToDoubleMatrix( mb );
- }
+ if( _mo != null ) { //CP ext function
+ MatrixBlock mb = _mo.acquireRead();
+ ret = DataConverter.convertToDoubleMatrix( mb );
+ _mo.release();
}
- catch(Exception ex)
- {
- throw new PackageRuntimeException(ex);
+ else { //traditional ext function (matrix file produced by reblock)
+ MatrixReader reader = MatrixReaderFactory.createMatrixReader(InputInfo.TextCellInputInfo);
+ MatrixBlock mb = reader.readMatrixFromHDFS(this.getFilePath(), _rows, _cols, -1, -1, -1);
+ ret = DataConverter.convertToDoubleMatrix( mb );
}
return ret;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/67e0d9d2/src/main/java/org/apache/sysml/udf/PackageFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/PackageFunction.java b/src/main/java/org/apache/sysml/udf/PackageFunction.java
index 39d23f4..3c01553 100644
--- a/src/main/java/org/apache/sysml/udf/PackageFunction.java
+++ b/src/main/java/org/apache/sysml/udf/PackageFunction.java
@@ -33,8 +33,7 @@ import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
*
*/
public abstract class PackageFunction implements Serializable
-{
-
+{
protected static final Log LOG = LogFactory.getLog(PackageFunction.class.getName());
private static final long serialVersionUID = 3274150928865462856L;
@@ -44,13 +43,8 @@ public abstract class PackageFunction implements Serializable
private String _baseDir; // base dir for all created files of that external function
private IDSequence _seq = null;
-
- /**
- * Constructor
- */
- public PackageFunction()
- {
+ public PackageFunction() {
_function_inputs = new ArrayList<FunctionParameter>();
_seq = new IDSequence();
}
@@ -62,7 +56,7 @@ public abstract class PackageFunction implements Serializable
*/
public final int getNumFunctionInputs() {
if (_function_inputs == null)
- throw new PackageRuntimeException("function inputs null");
+ throw new RuntimeException("function inputs null");
return (_function_inputs.size());
}
@@ -75,8 +69,7 @@ public abstract class PackageFunction implements Serializable
*/
public final FunctionParameter getFunctionInput(int pos) {
if (_function_inputs == null || _function_inputs.size() <= pos)
- throw new PackageRuntimeException(
- "function inputs null or size <= pos");
+ throw new RuntimeException("function inputs null or size <= pos");
return (_function_inputs.get(pos));
}
@@ -87,7 +80,6 @@ public abstract class PackageFunction implements Serializable
*
* @return
*/
-
public abstract int getNumFunctionOutputs();
/**
@@ -104,14 +96,12 @@ public abstract class PackageFunction implements Serializable
*
* @param numInputs
*/
-
public final void setNumFunctionInputs(int numInputs) {
if (_function_inputs == null)
- throw new PackageRuntimeException("function inputs null");
+ throw new RuntimeException("function inputs null");
if (_function_inputs.size() > numInputs)
- throw new PackageRuntimeException(
- "function inputs size > numInputs -- cannot reduce size");
+ throw new RuntimeException("function inputs size > numInputs -- cannot reduce size");
while (_function_inputs.size() < numInputs)
_function_inputs.add(null);
@@ -124,11 +114,9 @@ public abstract class PackageFunction implements Serializable
* @param input
* @param pos
*/
-
public final void setInput(FunctionParameter input, int pos) {
if (_function_inputs == null || _function_inputs.size() <= pos)
- throw new PackageRuntimeException(
- "function inputs null or size <= pos");
+ throw new RuntimeException("function inputs null or size <= pos");
_function_inputs.set(pos, input);
@@ -139,7 +127,6 @@ public abstract class PackageFunction implements Serializable
*
* @param fName
*/
-
public final void setConfiguration(String fName) {
_configurationFile = fName;
}
@@ -149,7 +136,6 @@ public abstract class PackageFunction implements Serializable
*
* @return
*/
-
public final String getConfiguration() {
return _configurationFile;
}
@@ -158,8 +144,7 @@ public abstract class PackageFunction implements Serializable
*
* @param dir
*/
- public void setBaseDir(String dir)
- {
+ public void setBaseDir(String dir) {
_baseDir = dir;
}
@@ -167,19 +152,17 @@ public abstract class PackageFunction implements Serializable
*
* @return
*/
- public String getBaseDir()
- {
+ public String getBaseDir() {
return _baseDir;
}
- public String createOutputFilePathAndName( String fname )
- {
+ public String createOutputFilePathAndName( String fname ) {
return _baseDir + fname + _seq.getNextID();
}
/**
- * Method that will be executed to perform this function.
+ * Method that will be executed to perform this function.
*/
public abstract void execute();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/67e0d9d2/src/main/java/org/apache/sysml/udf/PackageRuntimeException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/PackageRuntimeException.java b/src/main/java/org/apache/sysml/udf/PackageRuntimeException.java
deleted file mode 100644
index 1666bc4..0000000
--- a/src/main/java/org/apache/sysml/udf/PackageRuntimeException.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.udf;
-
-/**
- * Class to capture a runtime exception during package execution.
- *
- *
- *
- */
-public class PackageRuntimeException extends RuntimeException
-{
-
- private static final long serialVersionUID = 7388224928778587925L;
-
- public PackageRuntimeException(String msg) {
- super(msg);
- }
-
- public PackageRuntimeException(Exception e) {
- super(e);
- }
-
- public PackageRuntimeException(String msg, Exception e) {
- super(msg,e);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/67e0d9d2/src/main/java/org/apache/sysml/udf/lib/BinningWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/lib/BinningWrapper.java b/src/main/java/org/apache/sysml/udf/lib/BinningWrapper.java
index 5dd5879..2cb5ecb 100644
--- a/src/main/java/org/apache/sysml/udf/lib/BinningWrapper.java
+++ b/src/main/java/org/apache/sysml/udf/lib/BinningWrapper.java
@@ -22,7 +22,6 @@ package org.apache.sysml.udf.lib;
import org.apache.sysml.udf.FunctionParameter;
import org.apache.sysml.udf.Matrix;
import org.apache.sysml.udf.PackageFunction;
-import org.apache.sysml.udf.PackageRuntimeException;
import org.apache.sysml.udf.Scalar;
import org.apache.sysml.udf.Matrix.ValueType;
import org.apache.sysml.udf.Scalar.ScalarValueType;
@@ -55,7 +54,7 @@ public class BinningWrapper extends PackageFunction
case 1: return _defBins;
}
- throw new PackageRuntimeException("Invalid function output being requested");
+ throw new RuntimeException("Invalid function output being requested");
}
@Override
@@ -105,7 +104,7 @@ public class BinningWrapper extends PackageFunction
}
catch (Exception e)
{
- throw new PackageRuntimeException("Error executing external order function", e);
+ throw new RuntimeException("Error executing external order function", e);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/67e0d9d2/src/main/java/org/apache/sysml/udf/lib/DeNaNWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/lib/DeNaNWrapper.java b/src/main/java/org/apache/sysml/udf/lib/DeNaNWrapper.java
index 48946cc..5443893 100644
--- a/src/main/java/org/apache/sysml/udf/lib/DeNaNWrapper.java
+++ b/src/main/java/org/apache/sysml/udf/lib/DeNaNWrapper.java
@@ -23,13 +23,13 @@ package org.apache.sysml.udf.lib;
import org.apache.sysml.udf.FunctionParameter;
import org.apache.sysml.udf.Matrix;
import org.apache.sysml.udf.PackageFunction;
-import org.apache.sysml.udf.PackageRuntimeException;
import org.apache.sysml.udf.Matrix.ValueType;
/**
* Wrapper class to deNaN matrices by replacing all NaNs with zeros,
* made by modifying <code>OrderWrapper.java</code>
*/
+@Deprecated
public class DeNaNWrapper extends PackageFunction
{
@@ -51,7 +51,7 @@ public class DeNaNWrapper extends PackageFunction
if(pos == 0)
return ret;
- throw new PackageRuntimeException("Invalid function output being requested");
+ throw new RuntimeException("Invalid function output being requested");
}
@Override
@@ -73,7 +73,7 @@ public class DeNaNWrapper extends PackageFunction
}
catch (Exception e)
{
- throw new PackageRuntimeException("Error executing external removeNaN function", e);
+ throw new RuntimeException("Error executing external removeNaN function", e);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/67e0d9d2/src/main/java/org/apache/sysml/udf/lib/DeNegInfinityWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/lib/DeNegInfinityWrapper.java b/src/main/java/org/apache/sysml/udf/lib/DeNegInfinityWrapper.java
index af2efe7..d964c0a 100644
--- a/src/main/java/org/apache/sysml/udf/lib/DeNegInfinityWrapper.java
+++ b/src/main/java/org/apache/sysml/udf/lib/DeNegInfinityWrapper.java
@@ -23,13 +23,13 @@ package org.apache.sysml.udf.lib;
import org.apache.sysml.udf.FunctionParameter;
import org.apache.sysml.udf.Matrix;
import org.apache.sysml.udf.PackageFunction;
-import org.apache.sysml.udf.PackageRuntimeException;
import org.apache.sysml.udf.Matrix.ValueType;
/**
* Wrapper class to deNegInfinity matrices by replacing all Negative Infinities
* with zeros, made by modifying <code>DeNaNWrapper.java</code>
*/
+@Deprecated
public class DeNegInfinityWrapper extends PackageFunction
{
@@ -51,7 +51,7 @@ public class DeNegInfinityWrapper extends PackageFunction
if(pos == 0)
return ret;
- throw new PackageRuntimeException("Invalid function output being requested");
+ throw new RuntimeException("Invalid function output being requested");
}
@Override
@@ -73,7 +73,7 @@ public class DeNegInfinityWrapper extends PackageFunction
}
catch (Exception e)
{
- throw new PackageRuntimeException("Error executing external order function", e);
+ throw new RuntimeException("Error executing external order function", e);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/67e0d9d2/src/main/java/org/apache/sysml/udf/lib/DynamicProjectMatrixCP.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/lib/DynamicProjectMatrixCP.java b/src/main/java/org/apache/sysml/udf/lib/DynamicProjectMatrixCP.java
index 06d99bf..6d93e39 100644
--- a/src/main/java/org/apache/sysml/udf/lib/DynamicProjectMatrixCP.java
+++ b/src/main/java/org/apache/sysml/udf/lib/DynamicProjectMatrixCP.java
@@ -25,7 +25,6 @@ import org.apache.sysml.runtime.matrix.data.OutputInfo;
import org.apache.sysml.udf.FunctionParameter;
import org.apache.sysml.udf.Matrix;
import org.apache.sysml.udf.PackageFunction;
-import org.apache.sysml.udf.PackageRuntimeException;
import org.apache.sysml.udf.Matrix.ValueType;
public class DynamicProjectMatrixCP extends PackageFunction
@@ -99,7 +98,7 @@ public class DynamicProjectMatrixCP extends PackageFunction
}
catch(Exception e)
{
- throw new PackageRuntimeException("Error executing dynamic project of matrix", e);
+ throw new RuntimeException("Error executing dynamic project of matrix", e);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/67e0d9d2/src/main/java/org/apache/sysml/udf/lib/DynamicReadMatrixCP.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/lib/DynamicReadMatrixCP.java b/src/main/java/org/apache/sysml/udf/lib/DynamicReadMatrixCP.java
index 4147665..c386021 100644
--- a/src/main/java/org/apache/sysml/udf/lib/DynamicReadMatrixCP.java
+++ b/src/main/java/org/apache/sysml/udf/lib/DynamicReadMatrixCP.java
@@ -27,7 +27,6 @@ import org.apache.sysml.runtime.util.DataConverter;
import org.apache.sysml.udf.FunctionParameter;
import org.apache.sysml.udf.Matrix;
import org.apache.sysml.udf.PackageFunction;
-import org.apache.sysml.udf.PackageRuntimeException;
import org.apache.sysml.udf.Scalar;
import org.apache.sysml.udf.Matrix.ValueType;
@@ -80,7 +79,7 @@ public class DynamicReadMatrixCP extends PackageFunction
}
catch(Exception e)
{
- throw new PackageRuntimeException("Error executing dynamic read of matrix",e);
+ throw new RuntimeException("Error executing dynamic read of matrix",e);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/67e0d9d2/src/main/java/org/apache/sysml/udf/lib/DynamicReadMatrixRcCP.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/lib/DynamicReadMatrixRcCP.java b/src/main/java/org/apache/sysml/udf/lib/DynamicReadMatrixRcCP.java
index e96ee52..6077885 100644
--- a/src/main/java/org/apache/sysml/udf/lib/DynamicReadMatrixRcCP.java
+++ b/src/main/java/org/apache/sysml/udf/lib/DynamicReadMatrixRcCP.java
@@ -27,7 +27,6 @@ import org.apache.sysml.runtime.util.DataConverter;
import org.apache.sysml.udf.FunctionParameter;
import org.apache.sysml.udf.Matrix;
import org.apache.sysml.udf.PackageFunction;
-import org.apache.sysml.udf.PackageRuntimeException;
import org.apache.sysml.udf.Scalar;
import org.apache.sysml.udf.Matrix.ValueType;
import org.apache.sysml.udf.Scalar.ScalarValueType;
@@ -58,7 +57,7 @@ public class DynamicReadMatrixRcCP extends PackageFunction
if(pos == 1)
return _rc;
- throw new PackageRuntimeException("Invalid function output being requested");
+ throw new RuntimeException("Invalid function output being requested");
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/67e0d9d2/src/main/java/org/apache/sysml/udf/lib/DynamicWriteMatrixCP.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/lib/DynamicWriteMatrixCP.java b/src/main/java/org/apache/sysml/udf/lib/DynamicWriteMatrixCP.java
index 6085ab5..1674a2e 100644
--- a/src/main/java/org/apache/sysml/udf/lib/DynamicWriteMatrixCP.java
+++ b/src/main/java/org/apache/sysml/udf/lib/DynamicWriteMatrixCP.java
@@ -27,7 +27,6 @@ import org.apache.sysml.runtime.util.DataConverter;
import org.apache.sysml.udf.FunctionParameter;
import org.apache.sysml.udf.Matrix;
import org.apache.sysml.udf.PackageFunction;
-import org.apache.sysml.udf.PackageRuntimeException;
import org.apache.sysml.udf.Scalar;
import org.apache.sysml.udf.Scalar.ScalarValueType;
@@ -35,6 +34,7 @@ import org.apache.sysml.udf.Scalar.ScalarValueType;
*
*
*/
+@Deprecated
public class DynamicWriteMatrixCP extends PackageFunction
{
@@ -76,7 +76,7 @@ public class DynamicWriteMatrixCP extends PackageFunction
}
catch(Exception e)
{
- throw new PackageRuntimeException("Error executing dynamic write of matrix",e);
+ throw new RuntimeException("Error executing dynamic write of matrix",e);
}
_success = new Scalar(ScalarValueType.Boolean,String.valueOf(success));
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/67e0d9d2/src/main/java/org/apache/sysml/udf/lib/GatherWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/lib/GatherWrapper.java b/src/main/java/org/apache/sysml/udf/lib/GatherWrapper.java
index da217be..748d120 100644
--- a/src/main/java/org/apache/sysml/udf/lib/GatherWrapper.java
+++ b/src/main/java/org/apache/sysml/udf/lib/GatherWrapper.java
@@ -24,7 +24,6 @@ import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.udf.FunctionParameter;
import org.apache.sysml.udf.Matrix;
import org.apache.sysml.udf.PackageFunction;
-import org.apache.sysml.udf.PackageRuntimeException;
import org.apache.sysml.udf.Matrix.ValueType;
/**
* Wrapper class for conversions of bit vectors to condensed position vectors.
@@ -60,7 +59,7 @@ public class GatherWrapper extends PackageFunction
if(pos == 0)
return ret;
- throw new PackageRuntimeException("Invalid function output being requested");
+ throw new RuntimeException("Invalid function output being requested");
}
@Override
@@ -92,7 +91,7 @@ public class GatherWrapper extends PackageFunction
}
catch (Exception e)
{
- throw new PackageRuntimeException("Error executing external order function", e);
+ throw new RuntimeException("Error executing external order function", e);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/67e0d9d2/src/main/java/org/apache/sysml/udf/lib/OrderWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/lib/OrderWrapper.java b/src/main/java/org/apache/sysml/udf/lib/OrderWrapper.java
index 8c248ea..cca6c4d 100644
--- a/src/main/java/org/apache/sysml/udf/lib/OrderWrapper.java
+++ b/src/main/java/org/apache/sysml/udf/lib/OrderWrapper.java
@@ -25,13 +25,13 @@ import java.util.Comparator;
import org.apache.sysml.udf.FunctionParameter;
import org.apache.sysml.udf.Matrix;
import org.apache.sysml.udf.PackageFunction;
-import org.apache.sysml.udf.PackageRuntimeException;
import org.apache.sysml.udf.Scalar;
import org.apache.sysml.udf.Matrix.ValueType;
/**
* Wrapper class for Order rows based on values in a column
*
*/
+@Deprecated
public class OrderWrapper extends PackageFunction
{
@@ -53,7 +53,7 @@ public class OrderWrapper extends PackageFunction
if(pos == 0)
return ret;
- throw new PackageRuntimeException("Invalid function output being requested");
+ throw new RuntimeException("Invalid function output being requested");
}
@Override
@@ -79,7 +79,7 @@ public class OrderWrapper extends PackageFunction
}
catch (Exception e)
{
- throw new PackageRuntimeException("Error executing external order function", e);
+ throw new RuntimeException("Error executing external order function", e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/67e0d9d2/src/main/java/org/apache/sysml/udf/lib/PermutationMatrixWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/lib/PermutationMatrixWrapper.java b/src/main/java/org/apache/sysml/udf/lib/PermutationMatrixWrapper.java
index 6b1b67c..c4ce0c6 100644
--- a/src/main/java/org/apache/sysml/udf/lib/PermutationMatrixWrapper.java
+++ b/src/main/java/org/apache/sysml/udf/lib/PermutationMatrixWrapper.java
@@ -28,7 +28,6 @@ import org.apache.sysml.runtime.matrix.data.OutputInfo;
import org.apache.sysml.udf.FunctionParameter;
import org.apache.sysml.udf.Matrix;
import org.apache.sysml.udf.PackageFunction;
-import org.apache.sysml.udf.PackageRuntimeException;
import org.apache.sysml.udf.Scalar;
import org.apache.sysml.udf.Matrix.ValueType;
@@ -50,7 +49,7 @@ import org.apache.sysml.udf.Matrix.ValueType;
* false); B = P %*% A
*
*/
-
+@Deprecated
public class PermutationMatrixWrapper extends PackageFunction
{
@@ -70,7 +69,7 @@ public class PermutationMatrixWrapper extends PackageFunction
if (pos == 0)
return _ret;
- throw new PackageRuntimeException(
+ throw new RuntimeException(
"Invalid function output being requested");
}
@@ -110,9 +109,9 @@ public class PermutationMatrixWrapper extends PackageFunction
ValueType.Double);
_ret.setMatrixDoubleArray(mb, OutputInfo.BinaryBlockOutputInfo,
InputInfo.BinaryBlockInputInfo);
-
- } catch (Exception e) {
- throw new PackageRuntimeException(
+ }
+ catch (Exception e) {
+ throw new RuntimeException(
"Error executing external permutation_matrix function", e);
}
}
[2/4] incubator-systemml git commit: [SYSTEMML-570] Consolidated
buffer pool primitives matrices and frames
Posted by mb...@apache.org.
[SYSTEMML-570] Consolidated buffer pool primitives matrices and frames
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/b8152ea6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/b8152ea6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/b8152ea6
Branch: refs/heads/master
Commit: b8152ea6ac4053a64a07f5a53d1d27197d54c787
Parents: 1c476da
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Fri Apr 8 19:33:56 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Fri Apr 8 22:02:57 2016 -0700
----------------------------------------------------------------------
.../controlprogram/caching/CacheBlock.java | 6 +
.../controlprogram/caching/CacheableData.java | 164 +++++++++++--------
.../controlprogram/caching/FrameObject.java | 33 +---
.../controlprogram/caching/MatrixObject.java | 130 +--------------
.../cp/ComputationCPInstruction.java | 4 +-
.../sysml/runtime/matrix/data/FrameBlock.java | 4 +
.../sysml/runtime/matrix/data/MatrixBlock.java | 25 ++-
7 files changed, 134 insertions(+), 232 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b8152ea6/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheBlock.java
index 6cce9a6..fd70bb1 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheBlock.java
@@ -30,6 +30,12 @@ import org.apache.hadoop.io.Writable;
public interface CacheBlock extends Writable
{
/**
+ * Get the in-memory size in bytes of the cache block.
+ * @return
+ */
+ public long getInMemorySize();
+
+ /**
* Get the exact serialized size in bytes of the cache block.
* @return
*/
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b8152ea6/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
index be6f287..953f70b 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
@@ -44,8 +44,7 @@ import org.apache.sysml.runtime.util.LocalFileUtils;
* Under the protection of the envelope, the data blob may be evicted to
* the file system; then the subclass must set its reference to <code>null</code>
* to allow Java garbage collection. If other parts of the system continue
- * keep references to the data blob, its eviction will not release any memory.
- * To make the eviction meaningful, the rest of the system must dispose all references.
+ * keep references to the cache block, its eviction will not release any memory.
*/
public abstract class CacheableData<T extends CacheBlock> extends Data
{
@@ -60,8 +59,8 @@ public abstract class CacheableData<T extends CacheBlock> extends Data
public static final RPolicy CACHING_BUFFER_POLICY = RPolicy.FIFO;
public static final boolean CACHING_BUFFER_PAGECACHE = false;
public static final boolean CACHING_WRITE_CACHE_ON_READ = false;
- public static final String CACHING_COUNTER_GROUP_NAME = "SystemML Caching Counters";
- public static final String CACHEING_EVICTION_FILEEXTENSION = ".dat";
+ public static final String CACHING_COUNTER_GROUP_NAME = "SystemML Caching Counters";
+ public static final String CACHEING_EVICTION_FILEEXTENSION = ".dat";
/**
* Defines all possible cache status types for a data blob.
@@ -260,18 +259,18 @@ public abstract class CacheableData<T extends CacheBlock> extends Data
/**
*
- * @param props
+ * @return
*/
- public void setFileFormatProperties(FileFormatProperties props) {
- _formatProps = props;
+ public FileFormatProperties getFileFormatProperties() {
+ return _formatProps;
}
/**
*
- * @return
+ * @param props
*/
- public FileFormatProperties getFileFormatProperties() {
- return _formatProps;
+ public void setFileFormatProperties(FileFormatProperties props) {
+ _formatProps = props;
}
// --------- ABSTRACT HIGH-LEVEL CACHE I/O OPERATIONS ----------
@@ -319,9 +318,7 @@ public abstract class CacheableData<T extends CacheBlock> extends Data
*
* @throws CacheException
*/
- public synchronized void exportData()
- throws CacheException
- {
+ public synchronized void exportData() throws CacheException {
exportData( -1 );
}
@@ -333,8 +330,8 @@ public abstract class CacheableData<T extends CacheBlock> extends Data
*
* @throws CacheException
*/
- public synchronized void exportData( int replication )
- throws CacheException
+ public synchronized void exportData( int replication )
+ throws CacheException
{
exportData(_hdfsFileName, null, replication, null);
_hdfsFileExists = true;
@@ -388,39 +385,87 @@ public abstract class CacheableData<T extends CacheBlock> extends Data
* reference points to it;
* <code>false</code> if the blob reference is <code>null</code>.
*/
- protected abstract boolean isBlobPresent();
-
- /**
- * Low-level cache I/O method that physically evicts the data blob from
- * main memory. Must be defined by a subclass, never called by users.
- *
- * @param data
- *
- * @throws CacheException
- */
- protected abstract void evictBlobFromMemory(T data)
- throws CacheException;
-
+ protected boolean isBlobPresent() {
+ return (_data != null);
+ }
+
/**
* Low-level cache I/O method that physically restores the data blob to
* main memory. Must be defined by a subclass, never called by users.
*
* @throws CacheException
*/
- protected abstract void restoreBlobIntoMemory()
- throws CacheException;
+ protected void restoreBlobIntoMemory()
+ throws CacheException
+ {
+ String cacheFilePathAndName = getCacheFilePathAndName();
+ long begin = LOG.isTraceEnabled() ? System.currentTimeMillis() : 0;
+
+ if( LOG.isTraceEnabled() )
+ LOG.trace ("CACHE: Restoring matrix... " + getVarName() + " HDFS path: " +
+ (_hdfsFileName == null ? "null" : _hdfsFileName) + ", Restore from path: " + cacheFilePathAndName);
+
+ if (_data != null)
+ throw new CacheException (cacheFilePathAndName + " : Cannot restore on top of existing in-memory data.");
+ try {
+ _data = readBlobFromCache(cacheFilePathAndName);
+ }
+ catch (IOException e) {
+ throw new CacheException (cacheFilePathAndName + " : Restore failed.", e);
+ }
+
+ //check for success
+ if (_data == null)
+ throw new CacheException (cacheFilePathAndName + " : Restore failed.");
+
+ if( LOG.isTraceEnabled() )
+ LOG.trace("Restoring matrix - COMPLETED ... " + (System.currentTimeMillis()-begin) + " msec.");
+ }
+ /**
+ *
+ * @param fname
+ * @return
+ */
+ protected abstract T readBlobFromCache(String fname)
+ throws IOException;
+
/**
* Low-level cache I/O method that deletes the file containing the
* evicted data blob, without reading it.
* Must be defined by a subclass, never called by users.
*/
- protected abstract void freeEvictedBlob();
+ protected void freeEvictedBlob() {
+ String cacheFilePathAndName = getCacheFilePathAndName();
+ long begin = LOG.isTraceEnabled() ? System.currentTimeMillis() : 0;
+ if( LOG.isTraceEnabled() )
+ LOG.trace("CACHE: Freeing evicted matrix... " + getVarName() + " HDFS path: " +
+ (_hdfsFileName == null ? "null" : _hdfsFileName) + " Eviction path: " + cacheFilePathAndName);
+ LazyWriteBuffer.deleteBlock(cacheFilePathAndName);
+
+ if( LOG.isTraceEnabled() )
+ LOG.trace("Freeing evicted matrix - COMPLETED ... " + (System.currentTimeMillis()-begin) + " msec.");
+ }
+
+ /**
+ *
+ */
+ protected boolean isBelowCachingThreshold() {
+ return (_data.getInMemorySize() <= CACHING_THRESHOLD);
+ }
+
/**
*
*/
- protected abstract boolean isBelowCachingThreshold();
+ @Override //Data
+ public synchronized String getDebugName() {
+ int maxLength = 23;
+ String debugNameEnding = (_hdfsFileName == null ? "null" :
+ (_hdfsFileName.length() < maxLength ? _hdfsFileName : "..." +
+ _hdfsFileName.substring (_hdfsFileName.length() - maxLength + 3)));
+ return getVarName() + " " + debugNameEnding;
+ }
// ------------- IMPLEMENTED CACHE LOGIC METHODS --------------
@@ -541,58 +586,46 @@ public abstract class CacheableData<T extends CacheBlock> extends Data
// **************************************************
- public String getStatusAsString()
- {
+ public String getStatusAsString() {
return _cacheStatus.toString();
}
- //TODO isCached is only public for access from SparkExectionContext, once we can assume
- //the existence of spark libraries, we can move the related code to MatrixObject and
- //make this method protected again
- public boolean isCached(boolean inclCachedNoWrite)
- {
+ public boolean isCached(boolean inclCachedNoWrite) {
if( inclCachedNoWrite )
return (_cacheStatus == CacheStatus.CACHED || _cacheStatus == CacheStatus.CACHED_NOWRITE);
else
return (_cacheStatus == CacheStatus.CACHED);
}
- protected boolean isEmpty(boolean inclCachedNoWrite)
- {
+ protected boolean isEmpty(boolean inclCachedNoWrite) {
if( inclCachedNoWrite )
return (_cacheStatus == CacheStatus.EMPTY || _cacheStatus == CacheStatus.CACHED_NOWRITE);
else
return (_cacheStatus == CacheStatus.EMPTY);
}
- protected boolean isModify()
- {
+ protected boolean isModify() {
return (_cacheStatus == CacheStatus.MODIFY);
}
- protected void setEmpty()
- {
+ protected void setEmpty() {
_cacheStatus = CacheStatus.EMPTY;
}
- protected void setModify()
- {
+ protected void setModify() {
_cacheStatus = CacheStatus.MODIFY;
}
- protected void setCached()
- {
+ protected void setCached() {
_cacheStatus = CacheStatus.CACHED;
}
- protected void addOneRead()
- {
+ protected void addOneRead() {
_numReadThreads ++;
_cacheStatus = CacheStatus.READ;
}
- protected void removeOneRead(boolean doesBlobExist, boolean cacheNoWrite)
- {
+ protected void removeOneRead(boolean doesBlobExist, boolean cacheNoWrite) {
_numReadThreads --;
if (_numReadThreads == 0) {
if( cacheNoWrite )
@@ -604,21 +637,26 @@ public abstract class CacheableData<T extends CacheBlock> extends Data
}
}
- protected boolean isAvailableToRead()
- {
+ protected boolean isAvailableToRead() {
return ( _cacheStatus == CacheStatus.EMPTY
|| _cacheStatus == CacheStatus.CACHED
|| _cacheStatus == CacheStatus.CACHED_NOWRITE
|| _cacheStatus == CacheStatus.READ);
}
- protected boolean isAvailableToModify()
- {
+ protected boolean isAvailableToModify() {
return ( _cacheStatus == CacheStatus.EMPTY
|| _cacheStatus == CacheStatus.CACHED
|| _cacheStatus == CacheStatus.CACHED_NOWRITE);
}
+ // *******************************************
+ // *** ***
+ // *** LOW-LEVEL PRIVATE METHODS ***
+ // *** FOR SOFTREFERENCE CACHE ***
+ // *** ***
+ // *******************************************
+
/**
* Creates a new cache soft reference to the currently
* referenced cache block.
@@ -652,8 +690,7 @@ public abstract class CacheableData<T extends CacheBlock> extends Data
/**
*
*/
- public synchronized static void cleanupCacheDir()
- {
+ public synchronized static void cleanupCacheDir() {
//cleanup remaining cached writes
LazyWriteBuffer.cleanup();
@@ -725,18 +762,15 @@ public abstract class CacheableData<T extends CacheBlock> extends Data
_activeFlag = true; //turn on caching
}
- public static synchronized boolean isCachingActive()
- {
+ public static synchronized boolean isCachingActive() {
return _activeFlag;
}
- public static synchronized void disableCaching()
- {
+ public static synchronized void disableCaching() {
_activeFlag = false;
}
- public static synchronized void enableCaching()
- {
+ public static synchronized void enableCaching() {
_activeFlag = true;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b8152ea6/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
index 78dcdf5..0a014ed 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
@@ -20,6 +20,8 @@
package org.apache.sysml.runtime.controlprogram.caching;
+import java.io.IOException;
+
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
@@ -153,35 +155,10 @@ public class FrameObject extends CacheableData<FrameBlock>
}
////////////////////////////////////
- // currently unsupported caching api
-
- @Override
- protected boolean isBlobPresent() {
- throw new RuntimeException("Caching not implemented yet for FrameObject.");
- }
-
- @Override
- protected void evictBlobFromMemory(FrameBlock mb) throws CacheException {
- throw new RuntimeException("Caching not implemented yet for FrameObject.");
- }
-
- @Override
- protected void restoreBlobIntoMemory() throws CacheException {
- throw new RuntimeException("Caching not implemented yet for FrameObject.");
- }
-
- @Override
- protected void freeEvictedBlob() {
- throw new RuntimeException("Caching not implemented yet for FrameObject.");
- }
-
- @Override
- protected boolean isBelowCachingThreshold() {
- throw new RuntimeException("Caching not implemented yet for FrameObject.");
- }
+ // low-level cache API
@Override
- public String getDebugName() {
- throw new RuntimeException("Caching not implemented yet for FrameObject.");
+ protected FrameBlock readBlobFromCache(String fname) throws IOException {
+ return (FrameBlock)LazyWriteBuffer.readBlock(fname, false);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b8152ea6/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
index e05ea8d..f946633 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
@@ -574,7 +574,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
if( isCachingActive() //only if caching is enabled (otherwise keep everything in mem)
&& isCached(true) //not empty and not read/modify
- && !isUpdateInPlace() //pinned result variable
+ && !isUpdateInPlaceEnabled() //pinned result variable
&& !isBelowCachingThreshold() ) //min size for caching
{
if( write || _requiresLocalWrite )
@@ -1037,88 +1037,12 @@ public class MatrixObject extends CacheableData<MatrixBlock>
// *** ONLY CALLED BY THE SUPERCLASS ***
// *** ***
// *********************************************
-
-
- @Override
- protected boolean isBlobPresent()
- {
- return (_data != null);
- }
-
- @Override
- protected void evictBlobFromMemory ( MatrixBlock mb )
- throws CacheException
- {
- throw new CacheException("Redundant explicit eviction.");
- }
-
- @Override
- protected void restoreBlobIntoMemory ()
- throws CacheException
- {
- long begin = 0;
-
- if( LOG.isTraceEnabled() ) {
- LOG.trace("RESTORE of Matrix "+getVarName()+", "+_hdfsFileName);
- begin = System.currentTimeMillis();
- }
-
- String filePath = getCacheFilePathAndName();
- if( LOG.isTraceEnabled() )
- LOG.trace ("CACHE: Restoring matrix... " + getVarName() + " HDFS path: " +
- (_hdfsFileName == null ? "null" : _hdfsFileName) + ", Restore from path: " + filePath);
-
- if (_data != null)
- throw new CacheException (filePath + " : Cannot restore on top of existing in-memory data.");
-
- try
- {
- _data = readMatrix(filePath);
- }
- catch (IOException e)
- {
- throw new CacheException (filePath + " : Restore failed.", e);
- }
-
- //check for success
- if (_data == null)
- throw new CacheException (filePath + " : Restore failed.");
-
- if( LOG.isTraceEnabled() )
- LOG.trace("Restoring matrix - COMPLETED ... " + (System.currentTimeMillis()-begin) + " msec.");
- }
-
@Override
- protected void freeEvictedBlob()
- {
- String cacheFilePathAndName = getCacheFilePathAndName();
- long begin = 0;
- if( LOG.isTraceEnabled() ){
- LOG.trace("CACHE: Freeing evicted matrix... " + getVarName() + " HDFS path: " +
- (_hdfsFileName == null ? "null" : _hdfsFileName) + " Eviction path: " + cacheFilePathAndName);
- begin = System.currentTimeMillis();
- }
-
- LazyWriteBuffer.deleteBlock(cacheFilePathAndName);
-
- if( LOG.isTraceEnabled() )
- LOG.trace("Freeing evicted matrix - COMPLETED ... " + (System.currentTimeMillis()-begin) + " msec.");
+ protected MatrixBlock readBlobFromCache(String fname) throws IOException {
+ return (MatrixBlock)LazyWriteBuffer.readBlock(fname, true);
}
- @Override
- protected boolean isBelowCachingThreshold()
- {
- long rlen = _data.getNumRows();
- long clen = _data.getNumColumns();
- long nnz = _data.getNonZeros();
-
- //get in-memory size (assume dense, if nnz unknown)
- double sparsity = OptimizerUtils.getSparsity( rlen, clen, nnz );
- double size = MatrixBlock.estimateSizeInMemory( rlen, clen, sparsity );
-
- return ( !_data.isAllocated() || size <= CACHING_THRESHOLD );
- }
// *******************************************
// *** ***
@@ -1127,23 +1051,6 @@ public class MatrixObject extends CacheableData<MatrixBlock>
// *** ***
// *******************************************
- private boolean isUpdateInPlace()
- {
- return _updateInPlaceFlag;
- }
-
- /**
- *
- * @param filePathAndName
- * @return
- * @throws IOException
- */
- private MatrixBlock readMatrix (String filePathAndName)
- throws IOException
- {
- return (MatrixBlock)LazyWriteBuffer.readBlock(filePathAndName, true);
- }
-
/**
*
* @param filePathAndName
@@ -1399,25 +1306,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
return ret;
}
-
- @Override
- public synchronized String getDebugName()
- {
- int maxLength = 23;
- String debugNameEnding = (_hdfsFileName == null ? "null" :
- (_hdfsFileName.length() < maxLength ? _hdfsFileName : "..." +
- _hdfsFileName.substring (_hdfsFileName.length() - maxLength + 3)));
- return getVarName() + " " + debugNameEnding;
- }
-
- // *******************************************
- // *** ***
- // *** LOW-LEVEL PRIVATE METHODS ***
- // *** FOR SOFTREFERENCE CACHE ***
- // *** ***
- // *******************************************
-
/**
*
* @param add
@@ -1425,7 +1314,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
private void updateStatusPinned(boolean add) {
if( _data != null ) { //data should never be null
long size = sizePinned.get();
- size += (add ? 1 : -1) * _data.getSizeInMemory();
+ size += (add ? 1 : -1) * _data.getInMemorySize();
sizePinned.set( Math.max(size,0) );
}
}
@@ -1434,8 +1323,7 @@ public class MatrixObject extends CacheableData<MatrixBlock>
*
* @param flag
*/
- public void enableUpdateInPlace(boolean flag)
- {
+ public void enableUpdateInPlace(boolean flag) {
_updateInPlaceFlag = flag;
}
@@ -1443,18 +1331,14 @@ public class MatrixObject extends CacheableData<MatrixBlock>
*
* @return
*/
- public boolean isUpdateInPlaceEnabled()
- {
+ public boolean isUpdateInPlaceEnabled() {
return _updateInPlaceFlag;
}
-
-
/**
*
*/
- public void setEmptyStatus()
- {
+ public void setEmptyStatus() {
setEmpty();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b8152ea6/src/main/java/org/apache/sysml/runtime/instructions/cp/ComputationCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ComputationCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ComputationCPInstruction.java
index 93ee805..5824926 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ComputationCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ComputationCPInstruction.java
@@ -70,8 +70,8 @@ public abstract class ComputationCPInstruction extends CPInstruction
protected boolean checkGuardedRepresentationChange( MatrixBlock in1, MatrixBlock in2, MatrixBlock out )
{
double memDense = OptimizerUtils.estimateSize(out.getNumRows(), out.getNumColumns());
- double memIn1 = (in1 != null) ? in1.getSizeInMemory() : 0;
- double memIn2 = (in2 != null) ? in2.getSizeInMemory() : 0;
+ double memIn1 = (in1 != null) ? in1.getInMemorySize() : 0;
+ double memIn2 = (in2 != null) ? in2.getInMemorySize() : 0;
return ( memDense < memIn1 + memIn2 );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b8152ea6/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
index 5a4913b..7fca641 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
@@ -391,6 +391,10 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
////////
// CacheBlock implementation
+ public long getInMemorySize() {
+ return 1;
+ }
+
@Override
public long getExactSerializedSize() {
//TODO implement getExactSizeOnDisk();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b8152ea6/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
index 80b76c5..5ce255c 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
@@ -2504,20 +2504,6 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab
// Estimates size and sparsity
/**
- * Estimate size based on current sparse/dense representation.
- *
- * @return
- */
- public long getSizeInMemory()
- {
- double sp = OptimizerUtils.getSparsity(rlen, clen, nonZeros);
- if( sparse )
- return estimateSizeSparseInMemory(rlen, clen, sp);
- else
- return estimateSizeDenseInMemory(rlen, clen);
- }
-
- /**
*
* @return
*/
@@ -2773,6 +2759,17 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab
// CacheBlock implementation
@Override
+ public long getInMemorySize() {
+ //in-memory size given by header if not allocated
+ if( !isAllocated() )
+ return 44;
+ //in-memory size of dense/sparse representation
+ double sp = OptimizerUtils.getSparsity(rlen, clen, nonZeros);
+ return sparse ? estimateSizeSparseInMemory(rlen, clen, sp) :
+ estimateSizeDenseInMemory(rlen, clen);
+ }
+
+ @Override
public long getExactSerializedSize() {
return getExactSizeOnDisk();
}
[3/4] incubator-systemml git commit: [SYSTEMML-616] Cleanup
instruction handling (stage 2, remove redundant)
Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/67e0d9d2/src/main/java/org/apache/sysml/udf/lib/TimeWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/lib/TimeWrapper.java b/src/main/java/org/apache/sysml/udf/lib/TimeWrapper.java
index dd88ab3..f4802c8 100644
--- a/src/main/java/org/apache/sysml/udf/lib/TimeWrapper.java
+++ b/src/main/java/org/apache/sysml/udf/lib/TimeWrapper.java
@@ -24,7 +24,6 @@ package org.apache.sysml.udf.lib;
import org.apache.sysml.udf.FunctionParameter;
import org.apache.sysml.udf.PackageFunction;
-import org.apache.sysml.udf.PackageRuntimeException;
import org.apache.sysml.udf.Scalar;
import org.apache.sysml.udf.Scalar.ScalarValueType;
@@ -41,7 +40,6 @@ import org.apache.sysml.udf.Scalar.ScalarValueType;
*/
public class TimeWrapper extends PackageFunction
{
-
private static final long serialVersionUID = 1L;
private Scalar _ret;
@@ -56,21 +54,14 @@ public class TimeWrapper extends PackageFunction
if (pos == 0)
return _ret;
- throw new PackageRuntimeException(
+ throw new RuntimeException(
"Invalid function output being requested");
}
@Override
public void execute() {
- try {
- // int past = Integer.parseInt(((Scalar) getFunctionInput(0))
- // .getValue());
- long present = System.currentTimeMillis();
- _ret = new Scalar(ScalarValueType.Double, String.valueOf(present));
- } catch (Exception e) {
- throw new PackageRuntimeException(
- "Error executing external time function", e);
- }
+ long present = System.currentTimeMillis();
+ _ret = new Scalar(ScalarValueType.Double, String.valueOf(present));
}
}