You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2016/01/13 23:38:05 UTC
[1/2] incubator-systemml git commit: [SYSTEMML-454] Extended jmlc api
for variable reuse, incl tests/javadoc
Repository: incubator-systemml
Updated Branches:
refs/heads/master 231f69dd0 -> f8e25bb76
[SYSTEMML-454] Extended jmlc api for variable reuse, incl tests/javadoc
This change enables to reuse input variables (scalar/matrix) across
multiple executions of a prepared (precompiled) script, for example via
'setMatrix("W", obj, true)'. The extended API is flexible enough to bind
inputs to variables for a subset of executions as well as rebind inputs
between executions if necessary.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/7e0b41e8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/7e0b41e8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/7e0b41e8
Branch: refs/heads/master
Commit: 7e0b41e8549c38f26f7bbf9b3390b8e9b9b4d4c7
Parents: 231f69d
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Tue Jan 12 16:34:55 2016 -0800
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Tue Jan 12 16:42:40 2016 -0800
----------------------------------------------------------------------
.../org/apache/sysml/api/jmlc/Connection.java | 2 +-
.../apache/sysml/api/jmlc/PreparedScript.java | 158 ++++----
.../functions/jmlc/ReuseModelVariablesTest.java | 213 +++++++++++
src/test/scripts/functions/jmlc/m-svm-score.dml | 8 +-
.../functions/jmlc/reuse-glm-predict.dml | 381 +++++++++++++++++++
.../functions/jmlc/reuse-msvm-predict.dml | 74 ++++
6 files changed, 755 insertions(+), 81 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7e0b41e8/src/main/java/org/apache/sysml/api/jmlc/Connection.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/jmlc/Connection.java b/src/main/java/org/apache/sysml/api/jmlc/Connection.java
index a93053c..636a855 100644
--- a/src/main/java/org/apache/sysml/api/jmlc/Connection.java
+++ b/src/main/java/org/apache/sysml/api/jmlc/Connection.java
@@ -56,7 +56,7 @@ import org.apache.sysml.runtime.util.DataConverter;
*
* NOTES:
* * Currently fused API and implementation in order to reduce complexity.
- * * See SystemTMulticlassSVMScoreTest for an usage example.
+ * * See JUnit test cases (org.apache.sysml.test.integration.functions.jmlc) for examples.
*/
public class Connection
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7e0b41e8/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java b/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
index 892ae81..381f83c 100644
--- a/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
+++ b/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
@@ -19,7 +19,11 @@
package org.apache.sysml.api.jmlc;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
import java.util.HashSet;
+import java.util.Map.Entry;
import org.apache.sysml.api.DMLException;
import org.apache.sysml.conf.ConfigurationManager;
@@ -31,6 +35,7 @@ import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory;
import org.apache.sysml.runtime.instructions.cp.BooleanObject;
+import org.apache.sysml.runtime.instructions.cp.Data;
import org.apache.sysml.runtime.instructions.cp.DoubleObject;
import org.apache.sysml.runtime.instructions.cp.IntObject;
import org.apache.sysml.runtime.instructions.cp.ScalarObject;
@@ -49,19 +54,19 @@ import org.apache.sysml.runtime.util.DataConverter;
*/
public class PreparedScript
{
-
//input/output specification
private HashSet<String> _inVarnames = null;
private HashSet<String> _outVarnames = null;
+ private HashMap<String,Data> _inVarReuse = null;
//internal state (reused)
private Program _prog = null;
private LocalVariableMap _vars = null;
- /**
- * Meant to be invoked only from Connection
+ /**
+ * Meant to be invoked only from Connection
*/
- protected PreparedScript( Program prog, String[] inputs, String[] outputs )
+ protected PreparedScript( Program prog, String[] inputs, String[] outputs )
{
_prog = prog;
_vars = new LocalVariableMap();
@@ -73,100 +78,107 @@ public class PreparedScript
_outVarnames = new HashSet<String>();
for( String var : outputs )
_outVarnames.add( var );
+ _inVarReuse = new HashMap<String, Data>();
}
- /**
- *
- * @param varname
- * @param scalar
- * @throws DMLException
- */
- public void setScalar(String varname, ScalarObject scalar)
- throws DMLException
- {
- if( !_inVarnames.contains(varname) )
- throw new DMLException("Unspecified input variable: "+varname);
-
- _vars.put(varname, scalar);
+ /** Binds a scalar boolean to a registered input variable. */
+ public void setScalar(String varname, boolean scalar) throws DMLException {
+ setScalar(varname, scalar, false);
}
- /**
- *
- * @param varname
- * @param scalar
- * @throws DMLException
- */
- public void setScalar(String varname, boolean scalar)
- throws DMLException
- {
- if( !_inVarnames.contains(varname) )
- throw new DMLException("Unspecified input variable: "+varname);
-
- BooleanObject bo = new BooleanObject(varname, scalar);
- _vars.put(varname, bo);
+ /** Binds a scalar boolean to a registered input variable. */
+ public void setScalar(String varname, boolean scalar, boolean reuse) throws DMLException {
+ setScalar(varname, new BooleanObject(varname, scalar), reuse);
+ }
+
+ /** Binds a scalar long to a registered input variable. */
+ public void setScalar(String varname, long scalar) throws DMLException {
+ setScalar(varname, scalar, false);
+ }
+
+ /** Binds a scalar long to a registered input variable. */
+ public void setScalar(String varname, long scalar, boolean reuse) throws DMLException {
+ setScalar(varname, new IntObject(varname, scalar), reuse);
}
+ /** Binds a scalar double to a registered input variable. */
+ public void setScalar(String varname, double scalar) throws DMLException {
+ setScalar(varname, scalar, false);
+ }
+
+ /** Binds a scalar double to a registered input variable. */
+ public void setScalar(String varname, double scalar, boolean reuse) throws DMLException {
+ setScalar(varname, new DoubleObject(varname, scalar), reuse);
+ }
+
+ /** Binds a scalar string to a registered input variable. */
+ public void setScalar(String varname, String scalar) throws DMLException {
+ setScalar(varname, scalar, false);
+ }
+
+ /** Binds a scalar string to a registered input variable. */
+ public void setScalar(String varname, String scalar, boolean reuse) throws DMLException {
+ setScalar(varname, new StringObject(varname, scalar), reuse);
+ }
+
/**
+ * Binds a scalar object to a registered input variable.
+ * If reuse requested, then the input is guaranteed to be
+ * preserved over multiple <code>executeScript</code> calls.
*
* @param varname
* @param scalar
- * @throws DMLException
+ * @param reuse
+ * @throws DMLException
*/
- public void setScalar(String varname, long scalar)
+ public void setScalar(String varname, ScalarObject scalar, boolean reuse)
throws DMLException
{
if( !_inVarnames.contains(varname) )
throw new DMLException("Unspecified input variable: "+varname);
- IntObject io = new IntObject(varname, scalar);
- _vars.put(varname, io);
+ _vars.put(varname, scalar);
}
-
+
/**
*
* @param varname
- * @param scalar
- * @throws DMLException
+ * @param matrix
+ * @throws DMLException
*/
- public void setScalar(String varname, double scalar)
- throws DMLException
+ public void setMatrix(String varname, double[][] matrix)
+ throws DMLException
{
- if( !_inVarnames.contains(varname) )
- throw new DMLException("Unspecified input variable: "+varname);
-
- DoubleObject doo = new DoubleObject(varname, scalar);
- _vars.put(varname, doo);
+ setMatrix(varname, matrix, false);
}
/**
*
* @param varname
- * @param scalar
- * @throws DMLException
+ * @param matrix
+ * @throws DMLException
*/
- public void setScalar(String varname, String scalar)
+ public void setMatrix(String varname, double[][] matrix, boolean reuse)
throws DMLException
{
- if( !_inVarnames.contains(varname) )
- throw new DMLException("Unspecified input variable: "+varname);
-
- StringObject so = new StringObject(varname, scalar);
- _vars.put(varname, so);
+ setMatrix(varname, DataConverter.convertToMatrixBlock(matrix), reuse);
}
/**
+ * Binds a matrix object to a registered input variable.
+ * If reuse requested, then the input is guaranteed to be
+ * preserved over multiple <code>executeScript</code> calls.
*
* @param varname
* @param matrix
* @throws DMLException
*/
- public void setMatrix(String varname, MatrixBlock matrix)
+ public void setMatrix(String varname, MatrixBlock matrix, boolean reuse)
throws DMLException
{
if( !_inVarnames.contains(varname) )
throw new DMLException("Unspecified input variable: "+varname);
-
-
+
DMLConfig conf = ConfigurationManager.getConfig();
String scratch_space = conf.getTextValue(DMLConfig.SCRATCH_SPACE);
int blocksize = conf.getIntValue(DMLConfig.DEFAULT_BLOCK_SIZE);
@@ -180,26 +192,14 @@ public class PreparedScript
//put create matrix wrapper into symbol table
_vars.put(varname, mo);
+ if( reuse ) {
+ mo.enableCleanup(false); //prevent cleanup
+ _inVarReuse.put(varname, mo);
+ }
}
/**
- *
- * @param varname
- * @param matrix
- * @throws DMLException
- */
- public void setMatrix(String varname, double[][] matrix)
- throws DMLException
- {
- if( !_inVarnames.contains(varname) )
- throw new DMLException("Unspecified input variable: "+varname);
-
- MatrixBlock mb = DataConverter.convertToMatrixBlock(matrix);
- setMatrix(varname, mb);
- }
-
-
- /**
+ * Remove all current values bound to input or output variables.
*
*/
public void clearParameters()
@@ -208,6 +208,8 @@ public class PreparedScript
}
/**
+ * Executes the prepared script over the bound inputs, creating the
+ * result variables according to bound and registered outputs.
*
* @return
* @throws DMLException
@@ -215,6 +217,10 @@ public class PreparedScript
public ResultVariables executeScript()
throws DMLException
{
+ //add reused variables
+ for( Entry<String,Data> e : _inVarReuse.entrySet() )
+ _vars.put(e.getKey(), e.getValue());
+
//create and populate execution context
ExecutionContext ec = ExecutionContextFactory.createContext(_prog);
ec.setVariables(_vars);
@@ -222,6 +228,12 @@ public class PreparedScript
//core execute runtime program
_prog.execute( ec );
+ //cleanup unnecessary outputs
+ Collection<String> tmpVars = new ArrayList<String>(_vars.keySet());
+ for( String var : tmpVars )
+ if( !_outVarnames.contains(var) )
+ _vars.remove(var);
+
//construct results
ResultVariables rvars = new ResultVariables();
for( String ovar : _outVarnames )
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7e0b41e8/src/test/java/org/apache/sysml/test/integration/functions/jmlc/ReuseModelVariablesTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/jmlc/ReuseModelVariablesTest.java b/src/test/java/org/apache/sysml/test/integration/functions/jmlc/ReuseModelVariablesTest.java
new file mode 100644
index 0000000..2cc4449
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/jmlc/ReuseModelVariablesTest.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.jmlc;
+
+import java.io.IOException;
+import java.util.ArrayList;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.apache.sysml.api.DMLException;
+import org.apache.sysml.api.jmlc.Connection;
+import org.apache.sysml.api.jmlc.PreparedScript;
+import org.apache.sysml.api.jmlc.ResultVariables;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.Timing;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+
+/**
+ *
+ *
+ */
+public class ReuseModelVariablesTest extends AutomatedTestBase
+{
+ private final static String TEST_NAME1 = "reuse-glm-predict";
+ private final static String TEST_NAME2 = "reuse-msvm-predict";
+ private final static String TEST_DIR = "functions/jmlc/";
+ private final static String MODEL_FILE = "sentiment_model.mtx";
+ private final static String TEST_CLASS_DIR = TEST_DIR + ReuseModelVariablesTest.class.getSimpleName() + "/";
+
+ private final static int rows = 107;
+ private final static int cols = 46; //fixed
+
+ private final static int nRuns = 10;
+
+ private final static double sparsity1 = 0.7;
+ private final static double sparsity2 = 0.1;
+
+
+ @Override
+ public void setUp() {
+ addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "predicted_y" }) );
+ addTestConfiguration(TEST_NAME2, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] { "predicted_y" }) );
+ }
+
+ @Test
+ public void testJMLCScoreGLMDense() throws IOException {
+ runJMLCReuseTest(TEST_NAME1, false, false);
+ }
+
+ @Test
+ public void testJMLCScoreGLMSparse() throws IOException {
+ runJMLCReuseTest(TEST_NAME1, true, false);
+ }
+
+ @Test
+ public void testJMLCScoreGLMDenseReuse() throws IOException {
+ runJMLCReuseTest(TEST_NAME1, false, true);
+ }
+
+ @Test
+ public void testJMLCScoreGLMSparseReuse() throws IOException {
+ runJMLCReuseTest(TEST_NAME1, true, true);
+ }
+
+ @Test
+ public void testJMLCScoreMSVMDense() throws IOException {
+ runJMLCReuseTest(TEST_NAME2, false, false);
+ }
+
+ @Test
+ public void testJMLCScoreMSVMSparse() throws IOException {
+ runJMLCReuseTest(TEST_NAME2, true, false);
+ }
+
+ @Test
+ public void testJMLCScoreMSVMDenseReuse() throws IOException {
+ runJMLCReuseTest(TEST_NAME2, false, true);
+ }
+
+ @Test
+ public void testJMLCScoreMSVMSparseReuse() throws IOException {
+ runJMLCReuseTest(TEST_NAME2, true, true);
+ }
+
+ /**
+ *
+ * @param sparseM1
+ * @param sparseM2
+ * @param instType
+ * @throws IOException
+ */
+ private void runJMLCReuseTest( String testname, boolean sparse, boolean modelReuse )
+ throws IOException
+ {
+ String TEST_NAME = testname;
+
+ TestConfiguration config = getTestConfiguration(TEST_NAME);
+ loadTestConfiguration(config);
+
+ //generate inputs
+ ArrayList<double[][]> Xset = generateInputs(nRuns, rows, cols, sparse?sparsity2:sparsity1);
+
+ //run DML via JMLC
+ ArrayList<double[][]> Yset = execDMLScriptviaJMLC( TEST_NAME, Xset, modelReuse );
+
+ //check non-empty y
+ Assert.assertEquals(Xset.size(), Yset.size());
+ }
+
+ /**
+ *
+ * @param X
+ * @return
+ * @throws DMLException
+ * @throws IOException
+ */
+ private ArrayList<double[][]> execDMLScriptviaJMLC( String testname, ArrayList<double[][]> X, boolean modelReuse)
+ throws IOException
+ {
+ Timing time = new Timing(true);
+
+ ArrayList<double[][]> ret = new ArrayList<double[][]>();
+
+ //establish connection to SystemML
+ Connection conn = new Connection();
+
+ try
+ {
+ // For now, JMLC pipeline only allows dml
+ boolean parsePyDML = false;
+
+ //read and precompile script
+ String script = conn.readScript(SCRIPT_DIR + TEST_DIR + testname + ".dml");
+ PreparedScript pstmt = conn.prepareScript(script, new String[]{"X","W"}, new String[]{"predicted_y"}, parsePyDML);
+
+ //read model
+ String modelData = conn.readScript(SCRIPT_DIR + TEST_DIR + MODEL_FILE );
+ double[][] W = conn.convertToDoubleMatrix(modelData, rows, cols);
+
+ if( modelReuse )
+ pstmt.setMatrix("W", W, true);
+
+ //execute script multiple times
+ for( int i=0; i<nRuns; i++ )
+ {
+ //bind input parameters
+ if( !modelReuse )
+ pstmt.setMatrix("W", W);
+ pstmt.setMatrix("X", X.get(i));
+
+ //execute script
+ ResultVariables rs = pstmt.executeScript();
+
+ //get output parameter
+ double[][] Y = rs.getMatrix("predicted_y");
+ ret.add(Y); //keep result for comparison
+ }
+ }
+ catch(Exception ex)
+ {
+ ex.printStackTrace();
+ throw new IOException(ex);
+ }
+ finally
+ {
+ if( conn != null )
+ conn.close();
+ }
+
+ System.out.println("JMLC scoring w/ "+nRuns+" runs in "+time.stop()+"ms.");
+
+ return ret;
+ }
+
+ /**
+ *
+ * @param num
+ * @param rows
+ * @param cols
+ * @param sparsity
+ * @return
+ */
+ private ArrayList<double[][]> generateInputs( int num, int rows, int cols, double sparsity )
+ {
+ ArrayList<double[][]> ret = new ArrayList<double[][]>();
+
+ for( int i=0; i<num; i++ )
+ {
+ double[][] X = getRandomMatrix(rows, cols, -1, 1, sparsity, System.nanoTime());
+ ret.add(X);
+ }
+
+ return ret;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7e0b41e8/src/test/scripts/functions/jmlc/m-svm-score.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/jmlc/m-svm-score.dml b/src/test/scripts/functions/jmlc/m-svm-score.dml
index fb073ea..3b70100 100644
--- a/src/test/scripts/functions/jmlc/m-svm-score.dml
+++ b/src/test/scripts/functions/jmlc/m-svm-score.dml
@@ -19,13 +19,6 @@
#
#-------------------------------------------------------------
-# Note this script is externalized to customer, please do not change w/o consulting component owner.
-# Invocation command
-# hadoop jar SystemML.jar -f m-svm-score.dml -nvargs DATA=X
-# NUM_ROWS=<num_rows_in_X>
-# MODEL=W
-# PREDICTED=predicted_y
-
X = read("./tmp/X", rows=-1, cols=-1);
W = read("./tmp/W", rows=-1, cols=-1);
@@ -39,3 +32,4 @@ scores = X %*% W[1:n,] + ones %*% b;
predicted_y = rowIndexMax(scores);
write(predicted_y, "./tmp", format="text");
+
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7e0b41e8/src/test/scripts/functions/jmlc/reuse-glm-predict.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/jmlc/reuse-glm-predict.dml b/src/test/scripts/functions/jmlc/reuse-glm-predict.dml
new file mode 100644
index 0000000..e82939d
--- /dev/null
+++ b/src/test/scripts/functions/jmlc/reuse-glm-predict.dml
@@ -0,0 +1,381 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+# Default values for input parameters:
+fileM = ifdef ($M, " ");
+fileY = ifdef ($Y, " ");
+fileO = ifdef ($O, " ");
+fmtM = ifdef ($fmt, "text");
+
+dist_type = ifdef ($dfam, 1); # $dfam = 1;
+var_power = ifdef ($vpow, 0.0); # $vpow = 0.0;
+link_type = ifdef ($link, 0); # $link = 0;
+link_power = ifdef ($lpow, 1.0); # $lpow = 1.0;
+dispersion = ifdef ($disp, 1.0); # $disp = 1.0;
+
+var_power = as.double (var_power);
+link_power = as.double (link_power);
+dispersion = as.double (dispersion);
+
+if (dist_type == 3) {
+ link_type = 2;
+} else { if (link_type == 0) { # Canonical Link
+ if (dist_type == 1) {
+ link_type = 1;
+ link_power = 1.0 - var_power;
+ } else { if (dist_type == 2) {
+ link_type = 2;
+}}} }
+
+X = read ("./tmp/X", rows=-1, cols=-1);
+#X = table(X[,1], X[,2], X[,3])
+
+num_records = nrow (X);
+num_features = ncol (X);
+
+W = read ("./tmp/B", rows=-1, cols=-1);
+if (dist_type == 3) {
+ beta = W [1 : ncol (X), ];
+ intercept = W [nrow(W), ];
+} else {
+ beta = W [1 : ncol (X), 1];
+ intercept = W [nrow(W), 1];
+}
+if (nrow (W) == ncol (X)) {
+ intercept = 0.0 * intercept;
+ is_intercept = FALSE;
+} else {
+ num_features = num_features + 1;
+ is_intercept = TRUE;
+}
+
+ones_rec = matrix (1, rows = num_records, cols = 1);
+linear_terms = X %*% beta + ones_rec %*% intercept;
+[means, vars] =
+ glm_means_and_vars (linear_terms, dist_type, var_power, link_type, link_power);
+
+if (fileM != " ") {
+ write (means, fileM, format=fmtM);
+}
+
+predicted_y = rowIndexMax(means)
+write(predicted_y, "./tmp/P", format=fmtM)
+
+if (fileY != " ")
+{
+ Y = read (fileY);
+ ones_ctg = matrix (1, rows = ncol(Y), cols = 1);
+
+ # Statistics To Compute:
+
+ Z_logl = 0.0 / 0.0;
+ Z_logl_pValue = 0.0 / 0.0;
+ X2_pearson = 0.0 / 0.0;
+ df_pearson = -1;
+ G2_deviance = 0.0 / 0.0;
+ df_deviance = -1;
+ X2_pearson_pValue = 0.0 / 0.0;
+ G2_deviance_pValue = 0.0 / 0.0;
+ Z_logl_scaled = 0.0 / 0.0;
+ Z_logl_scaled_pValue = 0.0 / 0.0;
+ X2_scaled = 0.0 / 0.0;
+ X2_scaled_pValue = 0.0 / 0.0;
+ G2_scaled = 0.0 / 0.0;
+ G2_scaled_pValue = 0.0 / 0.0;
+
+ if (dist_type == 1 & link_type == 1) {
+ #
+ # POWER DISTRIBUTIONS (GAUSSIAN, POISSON, GAMMA, ETC.)
+ #
+ if (link_power == 0.0) {
+ is_zero_Y = ppred (Y, 0.0, "==");
+ lt_saturated = log (Y + is_zero_Y) - is_zero_Y / (1.0 - is_zero_Y);
+ } else {
+ lt_saturated = Y ^ link_power;
+ }
+ Y_counts = ones_rec;
+
+ X2_pearson = sum ((Y - means) ^ 2 / vars);
+ df_pearson = num_records - num_features;
+
+ log_l_part =
+ glm_partial_loglikelihood_for_power_dist_and_link (linear_terms, Y, var_power, link_power);
+ log_l_part_saturated =
+ glm_partial_loglikelihood_for_power_dist_and_link (lt_saturated, Y, var_power, link_power);
+
+ G2_deviance = 2 * sum (log_l_part_saturated) - 2 * sum (log_l_part);
+ df_deviance = num_records - num_features;
+
+ } else { if (dist_type >= 2) {
+ #
+ # BINOMIAL AND MULTINOMIAL DISTRIBUTIONS
+ #
+ if (ncol (Y) == 1) {
+ num_categories = ncol (beta) + 1;
+ if (min (Y) <= 0) {
+ # Category labels "0", "-1" etc. are converted into the baseline label
+ Y = Y + (- Y + num_categories) * ppred (Y, 0, "<=");
+ }
+ Y_size = min (num_categories, max(Y));
+ Y_unsized = table (seq (1, num_records, 1), Y);
+ Y = matrix (0, rows = num_records, cols = num_categories);
+ Y [, 1 : Y_size] = Y_unsized [, 1 : Y_size];
+ Y_counts = ones_rec;
+ } else {
+ Y_counts = rowSums (Y);
+ }
+
+ P = means;
+ zero_Y = ppred (Y, 0.0, "==");
+ zero_P = ppred (P, 0.0, "==");
+ ones_ctg = matrix (1, rows = ncol(Y), cols = 1);
+
+ logl_vec = rowSums (Y * log (P + zero_Y) );
+ ent1_vec = rowSums (P * log (P + zero_P) );
+ ent2_vec = rowSums (P * (log (P + zero_P))^2);
+ E_logl = sum (Y_counts * ent1_vec);
+ V_logl = sum (Y_counts * (ent2_vec - ent1_vec ^ 2));
+ Z_logl = (sum (logl_vec) - E_logl) / sqrt (V_logl);
+
+ means = means * (Y_counts %*% t(ones_ctg));
+ vars = vars * (Y_counts %*% t(ones_ctg));
+
+ frac_below_5 = sum (ppred (means, 5, "<")) / (nrow (means) * ncol (means));
+ frac_below_1 = sum (ppred (means, 1, "<")) / (nrow (means) * ncol (means));
+
+ if (frac_below_5 > 0.2 | frac_below_1 > 0.0) {
+ print ("WARNING: residual statistics are inaccurate here due to low cell means.");
+ }
+
+ X2_pearson = sum ((Y - means) ^ 2 / means);
+ df_pearson = (num_records - num_features) * (ncol(Y) - 1);
+
+ G2_deviance = 2 * sum (Y * log ((Y + zero_Y) / (means + zero_Y)));
+ df_deviance = (num_records - num_features) * (ncol(Y) - 1);
+ }}
+
+ if (Z_logl == Z_logl) {
+ Z_logl_absneg = - abs (Z_logl);
+ Z_logl_pValue = 2.0 * pnorm(target = Z_logl_absneg);
+ }
+ if (X2_pearson == X2_pearson & df_pearson > 0) {
+ X2_pearson_pValue = pchisq(target = X2_pearson, df = df_pearson, lower.tail=FALSE);
+ }
+ if (G2_deviance == G2_deviance & df_deviance > 0) {
+ G2_deviance_pValue = pchisq(target = G2_deviance, df = df_deviance, lower.tail=FALSE);
+ }
+
+ Z_logl_scaled = Z_logl / sqrt (dispersion);
+ X2_scaled = X2_pearson / dispersion;
+ G2_scaled = G2_deviance / dispersion;
+
+ if (Z_logl_scaled == Z_logl_scaled) {
+ Z_logl_scaled_absneg = - abs (Z_logl_scaled);
+ Z_logl_scaled_pValue = 2.0 * pnorm(target = Z_logl_scaled_absneg);
+ }
+ if (X2_scaled == X2_scaled & df_pearson > 0) {
+ X2_scaled_pValue = pchisq(target = X2_scaled, df = df_pearson, lower.tail=FALSE);
+ }
+ if (G2_scaled == G2_scaled & df_deviance > 0) {
+ G2_scaled_pValue = pchisq(target = G2_scaled, df = df_deviance, lower.tail=FALSE);
+ }
+
+ avg_tot_Y = colSums ( Y ) / sum (Y_counts);
+ avg_res_Y = colSums (Y - means) / sum (Y_counts);
+
+ ss_avg_tot_Y = colSums (( Y - Y_counts %*% avg_tot_Y) ^ 2);
+ ss_res_Y = colSums ((Y - means) ^ 2);
+ ss_avg_res_Y = colSums ((Y - means - Y_counts %*% avg_res_Y) ^ 2);
+
+ df_ss_res_Y = sum (Y_counts) - num_features;
+ if (is_intercept) {
+ df_ss_avg_res_Y = df_ss_res_Y;
+ } else {
+ df_ss_avg_res_Y = df_ss_res_Y - 1;
+ }
+
+ var_tot_Y = ss_avg_tot_Y / (sum (Y_counts) - 1);
+ if (df_ss_avg_res_Y > 0) {
+ var_res_Y = ss_avg_res_Y / df_ss_avg_res_Y;
+ } else {
+ var_res_Y = matrix (0.0, rows = 1, cols = ncol (Y)) / 0.0;
+ }
+ plain_R2_nobias = 1 - ss_avg_res_Y / ss_avg_tot_Y;
+ adjust_R2_nobias = 1 - var_res_Y / var_tot_Y;
+ plain_R2 = 1 - ss_res_Y / ss_avg_tot_Y;
+ if (df_ss_res_Y > 0) {
+ adjust_R2 = 1 - (ss_res_Y / df_ss_res_Y) / var_tot_Y;
+ } else {
+ adjust_R2 = matrix (0.0, rows = 1, cols = ncol (Y)) / 0.0;
+ }
+
+ predicted_avg_var_res_Y = dispersion * colSums (vars) / sum (Y_counts);
+
+ # PREPARING THE OUTPUT CSV STATISTICS FILE
+
+ str = "LOGLHOOD_Z,,FALSE," + Z_logl;
+ str = append (str, "LOGLHOOD_Z_PVAL,,FALSE," + Z_logl_pValue);
+ str = append (str, "PEARSON_X2,,FALSE," + X2_pearson);
+ str = append (str, "PEARSON_X2_BY_DF,,FALSE," + (X2_pearson / df_pearson));
+ str = append (str, "PEARSON_X2_PVAL,,FALSE," + X2_pearson_pValue);
+ str = append (str, "DEVIANCE_G2,,FALSE," + G2_deviance);
+ str = append (str, "DEVIANCE_G2_BY_DF,,FALSE," + (G2_deviance / df_deviance));
+ str = append (str, "DEVIANCE_G2_PVAL,,FALSE," + G2_deviance_pValue);
+ str = append (str, "LOGLHOOD_Z,,TRUE," + Z_logl_scaled);
+ str = append (str, "LOGLHOOD_Z_PVAL,,TRUE," + Z_logl_scaled_pValue);
+ str = append (str, "PEARSON_X2,,TRUE," + X2_scaled);
+ str = append (str, "PEARSON_X2_BY_DF,,TRUE," + (X2_scaled / df_pearson));
+ str = append (str, "PEARSON_X2_PVAL,,TRUE," + X2_scaled_pValue);
+ str = append (str, "DEVIANCE_G2,,TRUE," + G2_scaled);
+ str = append (str, "DEVIANCE_G2_BY_DF,,TRUE," + (G2_scaled / df_deviance));
+ str = append (str, "DEVIANCE_G2_PVAL,,TRUE," + G2_scaled_pValue);
+
+ for (i in 1:ncol(Y)) {
+ str = append (str, "AVG_TOT_Y," + i + ",," + castAsScalar (avg_tot_Y [1, i]));
+ str = append (str, "STDEV_TOT_Y," + i + ",," + castAsScalar (sqrt (var_tot_Y [1, i])));
+ str = append (str, "AVG_RES_Y," + i + ",," + castAsScalar (avg_res_Y [1, i]));
+ str = append (str, "STDEV_RES_Y," + i + ",," + castAsScalar (sqrt (var_res_Y [1, i])));
+ str = append (str, "PRED_STDEV_RES," + i + ",TRUE," + castAsScalar (sqrt (predicted_avg_var_res_Y [1, i])));
+ str = append (str, "PLAIN_R2," + i + ",," + castAsScalar (plain_R2 [1, i]));
+ str = append (str, "ADJUSTED_R2," + i + ",," + castAsScalar (adjust_R2 [1, i]));
+ str = append (str, "PLAIN_R2_NOBIAS," + i + ",," + castAsScalar (plain_R2_nobias [1, i]));
+ str = append (str, "ADJUSTED_R2_NOBIAS," + i + ",," + castAsScalar (adjust_R2_nobias [1, i]));
+ }
+
+ if (fileO != " ") {
+ write (str, fileO);
+ } else {
+ print (str);
+ }
+}
+
+glm_means_and_vars =
+ function (Matrix[double] linear_terms, int dist_type, double var_power, int link_type, double link_power)
+ return (Matrix[double] means, Matrix[double] vars)
+ # NOTE: "vars" represents the variance without dispersion, i.e. the V(mu) function.
+{
+ num_points = nrow (linear_terms);
+ if (dist_type == 1 & link_type == 1) {
+ # POWER DISTRIBUTION
+ if (link_power == 0.0) {
+ y_mean = exp (linear_terms);
+ } else { if (link_power == 1.0) {
+ y_mean = linear_terms;
+ } else { if (link_power == -1.0) {
+ y_mean = 1.0 / linear_terms;
+ } else {
+ y_mean = linear_terms ^ (1.0 / link_power);
+ }}}
+ if (var_power == 0.0) {
+ var_function = matrix (1.0, rows = num_points, cols = 1);
+ } else { if (var_power == 1.0) {
+ var_function = y_mean;
+ } else {
+ var_function = y_mean ^ var_power;
+ }}
+ means = y_mean;
+ vars = var_function;
+ } else { if (dist_type == 2 & link_type >= 1 & link_type <= 5) {
+ # BINOMIAL/BERNOULLI DISTRIBUTION
+ y_prob = matrix (0.0, rows = num_points, cols = 2);
+ if (link_type == 1 & link_power == 0.0) { # Binomial.log
+ y_prob [, 1] = exp (linear_terms);
+ y_prob [, 2] = 1.0 - y_prob [, 1];
+ } else { if (link_type == 1 & link_power != 0.0) { # Binomial.power_nonlog
+ y_prob [, 1] = linear_terms ^ (1.0 / link_power);
+ y_prob [, 2] = 1.0 - y_prob [, 1];
+ } else { if (link_type == 2) { # Binomial.logit
+ elt = exp (linear_terms);
+ y_prob [, 1] = elt / (1.0 + elt);
+ y_prob [, 2] = 1.0 / (1.0 + elt);
+ } else { if (link_type == 3) { # Binomial.probit
+ sign_lt = 2 * ppred (linear_terms, 0.0, ">=") - 1;
+ t_gp = 1.0 / (1.0 + abs (linear_terms) * 0.231641888); # 0.231641888 = 0.3275911 / sqrt (2.0)
+ erf_corr =
+ t_gp * ( 0.254829592
+ + t_gp * (-0.284496736 # "Handbook of Mathematical Functions", ed. by M. Abramowitz and I.A. Stegun,
+ + t_gp * ( 1.421413741 # U.S. Nat-l Bureau of Standards, 10th print (Dec 1972), Sec. 7.1.26, p. 299
+ + t_gp * (-1.453152027
+ + t_gp * 1.061405429)))) * sign_lt * exp (- (linear_terms ^ 2) / 2.0);
+ y_prob [, 1] = (1 + sign_lt) - erf_corr;
+ y_prob [, 2] = (1 - sign_lt) + erf_corr;
+ y_prob = y_prob / 2;
+ } else { if (link_type == 4) { # Binomial.cloglog
+ elt = exp (linear_terms);
+ is_too_small = ppred (10000000 + elt, 10000000, "==");
+ y_prob [, 2] = exp (- elt);
+ y_prob [, 1] = (1 - is_too_small) * (1.0 - y_prob [, 2]) + is_too_small * elt * (1.0 - elt / 2);
+ } else { if (link_type == 5) { # Binomial.cauchit
+ atan_linear_terms = atan (linear_terms);
+ y_prob [, 1] = 0.5 + atan_linear_terms / 3.1415926535897932384626433832795;
+ y_prob [, 2] = 0.5 - atan_linear_terms / 3.1415926535897932384626433832795;
+ }}}}}}
+ means = y_prob;
+ ones_ctg = matrix (1, rows = 2, cols = 1);
+ vars = means * (means %*% (1 - diag (ones_ctg)));
+ } else { if (dist_type == 3) {
+ # MULTINOMIAL LOGIT DISTRIBUTION
+ elt = exp (linear_terms);
+ ones_pts = matrix (1, rows = num_points, cols = 1);
+ elt = append (elt, ones_pts);
+ ones_ctg = matrix (1, rows = ncol (elt), cols = 1);
+ means = elt / (rowSums (elt) %*% t(ones_ctg));
+ vars = means * (means %*% (1 - diag (ones_ctg)));
+ } else {
+ means = matrix (0.0, rows = num_points, cols = 1);
+ vars = matrix (0.0, rows = num_points, cols = 1);
+} }}}
+
+glm_partial_loglikelihood_for_power_dist_and_link = # Assumes: dist_type == 1 & link_type == 1
+ function (Matrix[double] linear_terms, Matrix[double] Y, double var_power, double link_power)
+ return (Matrix[double] log_l_part)
+{
+ num_records = nrow (Y);
+ if (var_power == 1.0) { # Poisson
+ if (link_power == 0.0) { # Poisson.log
+ is_natural_parameter_log_zero = ppred (linear_terms, -1.0/0.0, "==");
+ natural_parameters = replace (target = linear_terms, pattern = -1.0/0.0, replacement = 0);
+ b_cumulant = exp (linear_terms);
+ } else { # Poisson.power_nonlog
+ is_natural_parameter_log_zero = ppred (linear_terms, 0.0, "==");
+ natural_parameters = log (linear_terms + is_natural_parameter_log_zero) / link_power;
+ b_cumulant = (linear_terms + is_natural_parameter_log_zero) ^ (1.0 / link_power) - is_natural_parameter_log_zero;
+ }
+ is_minus_infinity = ppred (Y, 0, ">") * is_natural_parameter_log_zero;
+ log_l_part = Y * natural_parameters - b_cumulant - is_minus_infinity / (1 - is_minus_infinity);
+ } else {
+ if (var_power == 2.0 & link_power == 0.0) { # Gamma.log
+ natural_parameters = - exp (- linear_terms);
+ b_cumulant = linear_terms;
+ } else { if (var_power == 2.0) { # Gamma.power_nonlog
+ natural_parameters = - linear_terms ^ (- 1.0 / link_power);
+ b_cumulant = log (linear_terms) / link_power;
+ } else { if (link_power == 0.0) { # PowerDist.log
+ natural_parameters = exp (linear_terms * (1.0 - var_power)) / (1.0 - var_power);
+ b_cumulant = exp (linear_terms * (2.0 - var_power)) / (2.0 - var_power);
+ } else { # PowerDist.power_nonlog
+ power_np = (1.0 - var_power) / link_power;
+ natural_parameters = (linear_terms ^ power_np) / (1.0 - var_power);
+ power_cu = (2.0 - var_power) / link_power;
+ b_cumulant = (linear_terms ^ power_cu) / (2.0 - var_power);
+ }}}
+ log_l_part = Y * natural_parameters - b_cumulant;
+} }
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7e0b41e8/src/test/scripts/functions/jmlc/reuse-msvm-predict.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/jmlc/reuse-msvm-predict.dml b/src/test/scripts/functions/jmlc/reuse-msvm-predict.dml
new file mode 100644
index 0000000..e572b11
--- /dev/null
+++ b/src/test/scripts/functions/jmlc/reuse-msvm-predict.dml
@@ -0,0 +1,74 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+cmdLine_Y = ifdef($Y, " ")
+cmdLine_confusion = ifdef($confusion, " ")
+cmdLine_accuracy = ifdef($accuracy, " ")
+cmdLine_scores = ifdef($scores, " ")
+cmdLine_fmt = ifdef($fmt, "text")
+
+X = read ("./tmp/X", rows=-1, cols=-1);
+N = max(X[,1]);
+
+W = read("./tmp/W", rows=-1, cols=-1);
+dimensions = as.scalar(W[nrow(W),1])
+intercept = as.scalar(W[nrow(W)-1,1])
+W = W[1:(nrow(W)-2),]
+
+#X = table(X[,1], X[,2], X[,3], N, dimensions)
+#print(nrow(X) + " " + ncol(X) + " " + dimensions)
+
+num_classes = ncol(W)
+m=ncol(X);
+
+b = matrix(0, rows=1, cols=num_classes)
+if (intercept == 1)
+ b = W[m+1,]
+
+ones = matrix(1, rows=N, cols=1)
+scores = X %*% W[1:m,] + ones %*% b;
+predicted_y = rowIndexMax(scores);
+
+if(sum(predicted_y) != 0)
+ write(predicted_y, cmdLine_scores, format=cmdLine_fmt);
+
+if(cmdLine_Y != " "){
+ y = read(cmdLine_Y);
+
+ if(min(y) < 1)
+ stop("Stopping due to invalid argument: Label vector (Y) must be recoded")
+
+ correct_percentage = sum(ppred(predicted_y - y, 0, "==")) / N * 100;
+
+ acc_str = "Accuracy (%): " + correct_percentage
+ print(acc_str)
+ if(cmdLine_accuracy != " ")
+ write(acc_str, cmdLine_accuracy)
+
+ num_classes_ground_truth = max(y)
+ if(num_classes < num_classes_ground_truth)
+ num_classes = num_classes_ground_truth
+
+ if(cmdLine_confusion != " "){
+ confusion_mat = table(predicted_y, y, num_classes, num_classes)
+ write(confusion_mat, cmdLine_confusion, format="csv")
+ }
+}
[2/2] incubator-systemml git commit: [SYSTEMML-330] Improved
jmlc/mlcontext argument/meta data handling
Posted by mb...@apache.org.
[SYSTEMML-330] Improved jmlc/mlcontext argument/meta data handling
This change introduces two new global parser configurations, which are
exploited by the jmlc/mlcontext apis. First,
AParserWrapper.IGNORE_UNSPECIFIED_ARGS is a global switch to ignore
unspecified script arguments by turning errors into warnings and
assigning default strings to these arguments. This enables to use
existing scripts without change through these apis. Second,
DataExpression.IGNORE_READ_WRITE_METADATA allows the jmlc api to
globally disable meta data reads which resolves a couple of corner cases
with existing meta data, etc.
Furthermore, this change leaves necessary parser refactoring and
consolidation tasks untouched. These issues will be resolved in separate
patches.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/f8e25bb7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/f8e25bb7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/f8e25bb7
Branch: refs/heads/master
Commit: f8e25bb76028e3f0526290a2e402b3bd7c0fb9df
Parents: 7e0b41e
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Wed Jan 13 10:01:15 2016 -0800
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Wed Jan 13 10:01:15 2016 -0800
----------------------------------------------------------------------
.../java/org/apache/sysml/api/MLContext.java | 23 +++++------
.../org/apache/sysml/api/jmlc/Connection.java | 15 +++++++-
.../org/apache/sysml/parser/AParserWrapper.java | 4 ++
.../org/apache/sysml/parser/DataExpression.java | 40 +++++++++++---------
.../parser/antlr4/DmlSyntacticValidator.java | 40 +++++++++++++-------
.../antlr4/DmlSyntacticValidatorHelper.java | 4 ++
.../parser/python/PydmlSyntacticValidator.java | 38 +++++++++++++------
.../python/PydmlSyntacticValidatorHelper.java | 5 ++-
.../functions/jmlc/reuse-glm-predict.dml | 8 ++--
.../functions/jmlc/reuse-msvm-predict.dml | 4 +-
10 files changed, 119 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f8e25bb7/src/main/java/org/apache/sysml/api/MLContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/MLContext.java b/src/main/java/org/apache/sysml/api/MLContext.java
index 6700f13..b89121e 100644
--- a/src/main/java/org/apache/sysml/api/MLContext.java
+++ b/src/main/java/org/apache/sysml/api/MLContext.java
@@ -1177,11 +1177,15 @@ public class MLContext {
throw new DMLRuntimeException("SystemML (and hence by definition MLContext) doesnot support parallel execute() calls from same or different MLContexts. "
+ "As a temporary fix, please do explicit synchronization, i.e. synchronized(MLContext.class) { ml.execute(...) } ");
}
- else {
- // Set active MLContext.
- _activeMLContext = this;
- }
+ // Set active MLContext.
+ _activeMLContext = this;
+
+ // Setup parser parameters
+ // TODO In the process of hardening mlcontext, we should also reinvestigate if we
+ // could be more restrictive and require known dimensions (rm REJECT_READ_WRITE_UNKNOWNS).
+ AParserWrapper.IGNORE_UNSPECIFIED_ARGS = true;
+ DataExpression.REJECT_READ_WRITE_UNKNOWNS = false;
if(_monitorUtils != null) {
_monitorUtils.resetMonitoringData();
@@ -1242,6 +1246,10 @@ public class MLContext {
finally {
// Reset active MLContext.
_activeMLContext = null;
+
+ // Reset parser parameters
+ AParserWrapper.IGNORE_UNSPECIFIED_ARGS = false;
+ DataExpression.REJECT_READ_WRITE_UNKNOWNS = true;
}
}
@@ -1288,17 +1296,12 @@ public class MLContext {
_monitorUtils.setDMLString(dmlScriptStr);
}
- DataExpression.REJECT_READ_UNKNOWN_SIZE = false;
-
//simplified compilation chain
_rtprog = null;
//parsing
AParserWrapper parser = AParserWrapper.createParser(parsePyDML);
DMLProgram prog = parser.parse(dmlScriptFilePath, dmlScriptStr, argVals);
- if(prog == null) {
- throw new ParseException("Couldnot parse the file:" + dmlScriptFilePath);
- }
//language validate
DMLTranslator dmlt = new DMLTranslator(prog);
@@ -1345,8 +1348,6 @@ public class MLContext {
ec.setVariables(inputSymbolTable);
}
- // System.out.println(Explain.explain(_rtprog));
-
//core execute runtime program
_rtprog.execute( ec );
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f8e25bb7/src/main/java/org/apache/sysml/api/jmlc/Connection.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/jmlc/Connection.java b/src/main/java/org/apache/sysml/api/jmlc/Connection.java
index 636a855..dd61235 100644
--- a/src/main/java/org/apache/sysml/api/jmlc/Connection.java
+++ b/src/main/java/org/apache/sysml/api/jmlc/Connection.java
@@ -70,7 +70,11 @@ public class Connection
public Connection()
{
//setup basic parameters for embedded execution
- DataExpression.REJECT_READ_UNKNOWN_SIZE = false;
+ //parser parameters
+ AParserWrapper.IGNORE_UNSPECIFIED_ARGS = true;
+ DataExpression.IGNORE_READ_WRITE_METADATA = true;
+ DataExpression.REJECT_READ_WRITE_UNKNOWNS = false;
+ //runtime parameters
DMLScript.rtplatform = RUNTIME_PLATFORM.SINGLE_NODE;
OptimizerUtils.PARALLEL_CP_READ_TEXTFORMATS = false;
OptimizerUtils.PARALLEL_CP_WRITE_TEXTFORMATS = false;
@@ -156,7 +160,14 @@ public class Connection
*/
public void close()
{
-
+ //reset parameters for embedded execution
+ AParserWrapper.IGNORE_UNSPECIFIED_ARGS = false;
+ DataExpression.IGNORE_READ_WRITE_METADATA = false;
+ DataExpression.REJECT_READ_WRITE_UNKNOWNS = true;
+ OptimizerUtils.PARALLEL_CP_READ_TEXTFORMATS = true;
+ OptimizerUtils.PARALLEL_CP_WRITE_TEXTFORMATS = true;
+ OptimizerUtils.PARALLEL_CP_READ_BINARYFORMATS = true;
+ OptimizerUtils.PARALLEL_CP_WRITE_BINARYFORMATS = true;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f8e25bb7/src/main/java/org/apache/sysml/parser/AParserWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/AParserWrapper.java b/src/main/java/org/apache/sysml/parser/AParserWrapper.java
index d3ded1f..fd600f2 100644
--- a/src/main/java/org/apache/sysml/parser/AParserWrapper.java
+++ b/src/main/java/org/apache/sysml/parser/AParserWrapper.java
@@ -30,6 +30,10 @@ import org.apache.sysml.parser.python.PyDMLParserWrapper;
*/
public abstract class AParserWrapper
{
+ //global parser configuration dml/pydml:
+ //1) skip errors on unspecified args (modified by mlcontext / jmlc)
+ public static boolean IGNORE_UNSPECIFIED_ARGS = false;
+
/**
*
* @param fileName
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f8e25bb7/src/main/java/org/apache/sysml/parser/DataExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/DataExpression.java b/src/main/java/org/apache/sysml/parser/DataExpression.java
index 5bd92ea..04e964b 100644
--- a/src/main/java/org/apache/sysml/parser/DataExpression.java
+++ b/src/main/java/org/apache/sysml/parser/DataExpression.java
@@ -43,8 +43,10 @@ import org.apache.sysml.utils.JSONHelper;
public class DataExpression extends DataIdentifier
{
-
- public static boolean REJECT_READ_UNKNOWN_SIZE = true;
+ //internal configuration (modified by mlcontext, jmlc apis)
+ //no read of meta data on mlcontext (local) /jmlc (global); ignore unknowns on jmlc
+ public static boolean IGNORE_READ_WRITE_METADATA = false; // global skip meta data reads
+ public static boolean REJECT_READ_WRITE_UNKNOWNS = true; // ignore missing meta data
public static final String RAND_ROWS = "rows";
public static final String RAND_COLS = "cols";
@@ -120,15 +122,16 @@ public class DataExpression extends DataIdentifier
private DataOp _opcode;
private HashMap<String, Expression> _varParams;
private boolean _strInit = false; //string initialize
-
- private boolean checkMetadata = true;
- public void setCheckMetadata(boolean checkMetadata) {
- this.checkMetadata = checkMetadata;
- }
+ private boolean _checkMetadata = true; // local skip meta data reads
public DataExpression(){
//do nothing
}
+
+
+ public void setCheckMetadata(boolean checkMetadata) {
+ _checkMetadata = checkMetadata;
+ }
public static DataExpression getDataExpression(String functionName, ArrayList<ParameterExpression> passedParamExprs,
String filename, int blp, int bcp, int elp, int ecp) throws DMLParseException {
@@ -469,8 +472,6 @@ public class DataExpression extends DataIdentifier
return _varParams.get(name);
}
-
-
public void addVarParam(String name, Expression value){
_varParams.put(name, value);
@@ -622,10 +623,10 @@ public class DataExpression extends DataIdentifier
String mtdFileName = getMTDFileName(inputFileName);
// track whether should attempt to read MTD file or not
- boolean shouldReadMTD = checkMetadata;
+ boolean shouldReadMTD = _checkMetadata && !IGNORE_READ_WRITE_METADATA;
// Check for file existence (before metadata parsing for meaningful error messages)
- if( shouldReadMTD && REJECT_READ_UNKNOWN_SIZE //skip check for jmlc/mlcontext apis
+ if( shouldReadMTD //skip check for jmlc/mlcontext
&& !MapReduceTool.existsFileOnHDFS(inputFileName))
{
String fsext = InfrastructureAnalyzer.isLocalMode() ? "FS (local mode)" : "HDFS";
@@ -892,19 +893,22 @@ public class DataExpression extends DataIdentifier
// initialize size of target data identifier to UNKNOWN
getOutput().setDimensions(-1, -1);
- if ( !isCSV && (getVarParam(READROWPARAM) == null || getVarParam(READCOLPARAM) == null)){
- raiseValidateError("Missing or incomplete dimension information in read statement: " + mtdFileName, conditional, LanguageErrorCodes.INVALID_PARAMETERS);
+ if ( !isCSV && REJECT_READ_WRITE_UNKNOWNS //skip check for csv format / jmlc api
+ && (getVarParam(READROWPARAM) == null || getVarParam(READCOLPARAM) == null) ) {
+ raiseValidateError("Missing or incomplete dimension information in read statement: "
+ + mtdFileName, conditional, LanguageErrorCodes.INVALID_PARAMETERS);
}
- if (getVarParam(READROWPARAM) instanceof ConstIdentifier && getVarParam(READCOLPARAM) instanceof ConstIdentifier) {
+ if (getVarParam(READROWPARAM) instanceof ConstIdentifier
+ && getVarParam(READCOLPARAM) instanceof ConstIdentifier)
+ {
// these are strings that are long values
Long dim1 = (getVarParam(READROWPARAM) == null) ? null : Long.valueOf( getVarParam(READROWPARAM).toString());
- Long dim2 = (getVarParam(READCOLPARAM) == null) ? null : Long.valueOf( getVarParam(READCOLPARAM).toString());
-
- if ( !isCSV && (dim1 <= 0 || dim2 <= 0) && REJECT_READ_UNKNOWN_SIZE )
- {
+ Long dim2 = (getVarParam(READCOLPARAM) == null) ? null : Long.valueOf( getVarParam(READCOLPARAM).toString());
+ if ( !isCSV && (dim1 <= 0 || dim2 <= 0) && REJECT_READ_WRITE_UNKNOWNS ) {
raiseValidateError("Invalid dimension information in read statement", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
}
+
// set dim1 and dim2 values
if (dim1 != null && dim2 != null){
getOutput().setDimensions(dim1, dim2);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f8e25bb7/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticValidator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticValidator.java b/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticValidator.java
index 27c62c2..8af3de9 100644
--- a/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticValidator.java
+++ b/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticValidator.java
@@ -29,7 +29,6 @@ import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.tree.ErrorNode;
import org.antlr.v4.runtime.tree.TerminalNode;
-
import org.apache.sysml.parser.ConditionalPredicate;
import org.apache.sysml.parser.DMLProgram;
import org.apache.sysml.parser.DataIdentifier;
@@ -86,6 +85,7 @@ import org.apache.sysml.parser.antlr4.DmlParser.TypedArgNoAssignContext;
import org.apache.sysml.parser.antlr4.DmlParser.UnaryExpressionContext;
import org.apache.sysml.parser.antlr4.DmlParser.ValueTypeContext;
import org.apache.sysml.parser.antlr4.DmlParser.WhileStatementContext;
+import org.apache.sysml.parser.AParserWrapper;
import org.apache.sysml.parser.AssignmentStatement;
import org.apache.sysml.parser.BinaryExpression;
import org.apache.sysml.parser.BooleanExpression;
@@ -117,6 +117,10 @@ import org.apache.sysml.parser.StatementBlock;
import org.apache.sysml.parser.StringIdentifier;
import org.apache.sysml.parser.WhileStatement;
+/**
+ * TODO: Refactor duplicated parser code dml/pydml (entire package).
+ *
+ */
public class DmlSyntacticValidator implements DmlListener
{
private DmlSyntacticValidatorHelper helper = null;
@@ -654,6 +658,7 @@ public class DmlSyntacticValidator implements DmlListener
}
}
+
private void fillExpressionInfoCommandLineParameters(String varName, ExpressionInfo dataInfo, Token start) {
if(!varName.startsWith("$")) {
@@ -689,29 +694,38 @@ public class DmlSyntacticValidator implements DmlListener
@Override
public void exitCommandlineParamExpression(CommandlineParamExpressionContext ctx) {
- String varName = ctx.getText().trim();
- fillExpressionInfoCommandLineParameters(varName, ctx.dataInfo, ctx.start);
- if(ctx.dataInfo.expr == null) {
- // Check if the parent is ifdef
- if(!(ctx.parent instanceof IfdefAssignmentStatementContext)) {
- helper.notifyErrorListeners("the parameter " + varName + " either needs to be passed through commandline or initialized to default value", ctx.start);
- }
- }
+ handleCommandlineArgumentExpression(ctx);
}
@Override
public void exitCommandlinePositionExpression(CommandlinePositionExpressionContext ctx) {
- String varName = ctx.getText().trim();
+ handleCommandlineArgumentExpression(ctx);
+ }
+
+ /**
+ *
+ * @param ctx
+ */
+ private void handleCommandlineArgumentExpression(DataIdentifierContext ctx)
+ {
+ String varName = ctx.getText().trim();
fillExpressionInfoCommandLineParameters(varName, ctx.dataInfo, ctx.start);
+
if(ctx.dataInfo.expr == null) {
- // Check if the parent is ifdef
if(!(ctx.parent instanceof IfdefAssignmentStatementContext)) {
- helper.notifyErrorListeners("the parameter " + varName + " either needs to be passed through commandline or initialized to default value", ctx.start);
+ String msg = "The parameter " + varName + " either needs to be passed "
+ + "through commandline or initialized to default value.";
+ if( AParserWrapper.IGNORE_UNSPECIFIED_ARGS ) {
+ ctx.dataInfo.expr = getConstIdFromString(" ", ctx.start);
+ helper.raiseWarning(msg, ctx.start);
+ }
+ else {
+ helper.notifyErrorListeners(msg, ctx.start);
+ }
}
}
}
-
// --------------------------------------------------------------------
@Override
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f8e25bb7/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticValidatorHelper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticValidatorHelper.java b/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticValidatorHelper.java
index 3c51b9e..91dbbf0 100644
--- a/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticValidatorHelper.java
+++ b/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticValidatorHelper.java
@@ -45,6 +45,10 @@ public class DmlSyntacticValidatorHelper {
this._errorListener.validationError(op.getLine(), op.getCharPositionInLine(), message);
}
+ public void raiseWarning(String message, Token op) {
+ this._errorListener.validationWarning(op.getLine(), op.getCharPositionInLine(), message);
+ }
+
public String getCurrentFileName() {
return _errorListener.peekFileName();
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f8e25bb7/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidator.java b/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidator.java
index fc56fc2..e7a2b6e 100644
--- a/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidator.java
+++ b/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidator.java
@@ -46,6 +46,7 @@ import org.apache.sysml.parser.Expression;
import org.apache.sysml.parser.Expression.DataOp;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.parser.AParserWrapper;
import org.apache.sysml.parser.ExternalFunctionStatement;
import org.apache.sysml.parser.ForStatement;
import org.apache.sysml.parser.FunctionCallIdentifier;
@@ -120,6 +121,10 @@ import org.apache.sysml.parser.python.PydmlParser.UnaryExpressionContext;
import org.apache.sysml.parser.python.PydmlParser.ValueDataTypeCheckContext;
import org.apache.sysml.parser.python.PydmlParser.WhileStatementContext;
+/**
+ * TODO: Refactor duplicated parser code dml/pydml (entire package).
+ *
+ */
public class PydmlSyntacticValidator implements PydmlListener
{
private PydmlSyntacticValidatorHelper helper = null;
@@ -664,29 +669,38 @@ public class PydmlSyntacticValidator implements PydmlListener
@Override
public void exitCommandlineParamExpression(CommandlineParamExpressionContext ctx) {
- String varName = ctx.getText().trim();
- fillExpressionInfoCommandLineParameters(varName, ctx.dataInfo, ctx.start);
- if(ctx.dataInfo.expr == null) {
- // Check if the parent is ifdef
- if(!(ctx.parent instanceof IfdefAssignmentStatementContext)) {
- helper.notifyErrorListeners("the parameter " + varName + " either needs to be passed through commandline or initialized to default value", ctx.start);
- }
- }
+ handleCommandlineArgumentExpression(ctx);
}
@Override
public void exitCommandlinePositionExpression(CommandlinePositionExpressionContext ctx) {
- String varName = ctx.getText().trim();
+ handleCommandlineArgumentExpression(ctx);
+ }
+
+ /**
+ *
+ * @param ctx
+ */
+ private void handleCommandlineArgumentExpression(DataIdentifierContext ctx)
+ {
+ String varName = ctx.getText().trim();
fillExpressionInfoCommandLineParameters(varName, ctx.dataInfo, ctx.start);
+
if(ctx.dataInfo.expr == null) {
- // Check if the parent is ifdef
if(!(ctx.parent instanceof IfdefAssignmentStatementContext)) {
- helper.notifyErrorListeners("the parameter " + varName + " either needs to be passed through commandline or initialized to default value", ctx.start);
+ String msg = "The parameter " + varName + " either needs to be passed "
+ + "through commandline or initialized to default value.";
+ if( AParserWrapper.IGNORE_UNSPECIFIED_ARGS ) {
+ ctx.dataInfo.expr = getConstIdFromString(" ", ctx.start);
+ helper.raiseWarning(msg, ctx.start);
+ }
+ else {
+ helper.notifyErrorListeners(msg, ctx.start);
+ }
}
}
}
-
// --------------------------------------------------------------------
@Override
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f8e25bb7/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidatorHelper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidatorHelper.java b/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidatorHelper.java
index 67aeb4a..f014684 100644
--- a/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidatorHelper.java
+++ b/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidatorHelper.java
@@ -23,7 +23,6 @@ import java.util.ArrayList;
import java.util.List;
import org.antlr.v4.runtime.Token;
-
import org.apache.sysml.parser.DMLProgram;
import org.apache.sysml.parser.python.PydmlParser.FunctionCallAssignmentStatementContext;
import org.apache.sysml.parser.python.PydmlParser.ParameterizedExpressionContext;
@@ -43,6 +42,10 @@ public class PydmlSyntacticValidatorHelper
public void notifyErrorListeners(String message, Token op) {
this._errorListener.validationError(op.getLine(), op.getCharPositionInLine(), message);
}
+
+ public void raiseWarning(String message, Token op) {
+ this._errorListener.validationWarning(op.getLine(), op.getCharPositionInLine(), message);
+ }
public String getCurrentFileName() {
return _errorListener.peekFileName();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f8e25bb7/src/test/scripts/functions/jmlc/reuse-glm-predict.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/jmlc/reuse-glm-predict.dml b/src/test/scripts/functions/jmlc/reuse-glm-predict.dml
index e82939d..540971a 100644
--- a/src/test/scripts/functions/jmlc/reuse-glm-predict.dml
+++ b/src/test/scripts/functions/jmlc/reuse-glm-predict.dml
@@ -20,6 +20,8 @@
#-------------------------------------------------------------
# Default values for input parameters:
+fileX = $X;
+fileB = $B;
fileM = ifdef ($M, " ");
fileY = ifdef ($Y, " ");
fileO = ifdef ($O, " ");
@@ -45,13 +47,13 @@ if (dist_type == 3) {
link_type = 2;
}}} }
-X = read ("./tmp/X", rows=-1, cols=-1);
+X = read(fileX);
#X = table(X[,1], X[,2], X[,3])
num_records = nrow (X);
num_features = ncol (X);
-W = read ("./tmp/B", rows=-1, cols=-1);
+W = read (fileB);
if (dist_type == 3) {
beta = W [1 : ncol (X), ];
intercept = W [nrow(W), ];
@@ -77,7 +79,7 @@ if (fileM != " ") {
}
predicted_y = rowIndexMax(means)
-write(predicted_y, "./tmp/P", format=fmtM)
+write(predicted_y, $P, format=fmtM)
if (fileY != " ")
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f8e25bb7/src/test/scripts/functions/jmlc/reuse-msvm-predict.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/jmlc/reuse-msvm-predict.dml b/src/test/scripts/functions/jmlc/reuse-msvm-predict.dml
index e572b11..8bd41e1 100644
--- a/src/test/scripts/functions/jmlc/reuse-msvm-predict.dml
+++ b/src/test/scripts/functions/jmlc/reuse-msvm-predict.dml
@@ -25,10 +25,10 @@ cmdLine_accuracy = ifdef($accuracy, " ")
cmdLine_scores = ifdef($scores, " ")
cmdLine_fmt = ifdef($fmt, "text")
-X = read ("./tmp/X", rows=-1, cols=-1);
+X = read ($X);
N = max(X[,1]);
-W = read("./tmp/W", rows=-1, cols=-1);
+W = read($W);
dimensions = as.scalar(W[nrow(W),1])
intercept = as.scalar(W[nrow(W)-1,1])
W = W[1:(nrow(W)-2),]