You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by lr...@apache.org on 2015/12/03 19:46:18 UTC
[47/78] [abbrv] [partial] incubator-systemml git commit: Move files
to new package folder structure
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/api/python/SystemML.py
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/api/python/SystemML.py b/src/main/java/com/ibm/bi/dml/api/python/SystemML.py
deleted file mode 100644
index a4b22ff..0000000
--- a/src/main/java/com/ibm/bi/dml/api/python/SystemML.py
+++ /dev/null
@@ -1,198 +0,0 @@
-#!/usr/bin/python
-
-from py4j.protocol import Py4JJavaError, Py4JError
-import traceback
-import os
-from pyspark.sql import *
-from pyspark.rdd import RDD
-
-
-class MLContext(object):
-
- """
- Simple wrapper class for MLContext in SystemML.jar
- ...
- Attributes
- ----------
- ml : MLContext
- A reference to the java MLContext
- sc : SparkContext
- The SparkContext that has been specified during initialization
- """
-
- def __init__(self, sc, *args):
- """
- If initialized with a SparkContext, will connect to the Java MLContext
- class.
- args:
- sc (SparkContext): the current SparkContext
- monitor (boolean=False): Whether to monitor the performance
- forceSpark (boolean=False): Whether to force execution on spark
- returns:
- MLContext: Instance of MLContext
- """
-
- try:
- monitorPerformance = (args[0] if len(args) > 0 else False)
- setForcedSparkExecType = (args[1] if len(args) > 1 else False)
- self.sc = sc
- self.ml = sc._jvm.com.ibm.bi.dml.api.MLContext(sc._jsc, monitorPerformance, setForcedSparkExecType)
- except Py4JError:
- traceback.print_exc()
-
- def reset(self):
- """
- Call this method of you want to clear any RDDs set via
- registerInput or registerOutput
- """
- try:
- self.ml.reset()
- except Py4JJavaError:
- traceback.print_exc()
-
- def execute(self, dmlScriptFilePath, *args):
- """
- Executes the script in spark-mode by passing the arguments to the
- MLContext java class.
- Returns:
- MLOutput: an instance of the MLOutput-class
- """
- numArgs = len(args) + 1
- try:
- if numArgs == 1:
- jmlOut = self.ml.execute(dmlScriptFilePath)
- mlOut = MLOutput(jmlOut, self.sc)
- return mlOut
- elif numArgs == 2:
- jmlOut = self.ml.execute(dmlScriptFilePath, args[0])
- mlOut = MLOutput(jmlOut, self.sc)
- return mlOut
- elif numArgs == 3:
- jmlOut = self.ml.execute(dmlScriptFilePath, args[0], args[1])
- mlOut = MLOutput(jmlOut, self.sc)
- return mlOut
- elif numArgs == 4:
- jmlOut = self.ml.execute(dmlScriptFilePath, args[0], args[1], args[2])
- mlOut = MLOutput(jmlOut, self.sc)
- return mlOut
- else:
- raise TypeError('Arguments do not match MLContext-API')
- except Py4JJavaError:
- traceback.print_exc()
-
- def registerInput(self, varName, src, *args):
- """
- Method to register inputs used by the DML script.
- Supported format:
- 1. DataFrame
- 2. CSV/Text (as JavaRDD<String> or JavaPairRDD<LongWritable, Text>)
- 3. Binary blocked RDD (JavaPairRDD<MatrixIndexes,MatrixBlock>))
- Also overloaded to support metadata information such as format, rlen, clen, ...
- Please note the variable names given below in quotes correspond to the variables in DML script.
- These variables need to have corresponding read/write associated in DML script.
- Currently, only matrix variables are supported through registerInput/registerOutput interface.
- To pass scalar variables, use named/positional arguments (described later) or wrap them into matrix variable.
- """
- numArgs = len(args) + 2
-
- if hasattr(src, '_jdf'):
- rdd = src._jdf
- elif hasattr(src, '_jrdd'):
- rdd = src._jrdd
- else:
- rdd = src
-
- try:
- if numArgs == 2:
- self.ml.registerInput(varName, rdd)
- elif numArgs == 3:
- self.ml.registerInput(varName, rdd, args[0])
- elif numArgs == 4:
- self.ml.registerInput(varName, rdd, args[0], args[1])
- elif numArgs == 5:
- self.ml.registerInput(varName, rdd, args[0], args[1], args[2])
- elif numArgs == 6:
- self.ml.registerInput(varName, rdd, args[0], args[1], args[2], args[3])
- elif numArgs == 7:
- self.ml.registerInput(varName, rdd, args[0], args[1], args[2], args[3], args[4])
- elif numArgs == 10:
- self.ml.registerInput(varName, rdd, args[0], args[1], args[2], args[3], args[4], args[5], args[6], args[7])
- else:
- raise TypeError('Arguments do not match MLContext-API')
- except Py4JJavaError:
-
- traceback.print_exc()
-
- def registerOutput(self, varName):
- """
- Register output variables used in the DML script
- args:
- varName: (String) The name used in the DML script
- """
-
- try:
- self.ml.registerOutput(varName)
- except Py4JJavaError:
- traceback.print_exc()
-
- def getDmlJson(self):
- try:
- return self.ml.getMonitoringUtil().getRuntimeInfoInJSONFormat()
- except Py4JJavaError:
- traceback.print_exc()
-
-
-class MLOutput(object):
-
- """
- This is a simple wrapper object that returns the output of execute from MLContext
- ...
- Attributes
- ----------
- jmlOut MLContext:
- A reference to the MLOutput object through py4j
- """
-
- def __init__(self, jmlOut, sc):
- self.jmlOut = jmlOut
- self.sc = sc
-
- def getBinaryBlockedRDD(self, varName):
- raise Exception('Not supported in Python MLContext')
- #try:
- # rdd = RDD(self.jmlOut.getBinaryBlockedRDD(varName), self.sc)
- # return rdd
- #except Py4JJavaError:
- # traceback.print_exc()
-
- def getMatrixCharacteristics(self, varName):
- raise Exception('Not supported in Python MLContext')
- #try:
- # chars = self.jmlOut.getMatrixCharacteristics(varName)
- # return chars
- #except Py4JJavaError:
- # traceback.print_exc()
-
- def getDF(self, sqlContext, varName):
- try:
- jdf = self.jmlOut.getDF(sqlContext._scala_SQLContext, varName)
- df = DataFrame(jdf, sqlContext)
- return df
- except Py4JJavaError:
- traceback.print_exc()
-
- def getMLMatrix(self, sqlContext, varName):
- raise Exception('Not supported in Python MLContext')
- #try:
- # mlm = self.jmlOut.getMLMatrix(sqlContext._scala_SQLContext, varName)
- # return mlm
- #except Py4JJavaError:
- # traceback.print_exc()
-
- def getStringRDD(self, varName, format):
- raise Exception('Not supported in Python MLContext')
- #try:
- # rdd = RDD(self.jmlOut.getStringRDD(varName, format), self.sc)
- # return rdd
- #except Py4JJavaError:
- # traceback.print_exc()
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/conf/ConfigurationManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/conf/ConfigurationManager.java b/src/main/java/com/ibm/bi/dml/conf/ConfigurationManager.java
deleted file mode 100644
index a8fa5f6..0000000
--- a/src/main/java/com/ibm/bi/dml/conf/ConfigurationManager.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.conf;
-
-import org.apache.hadoop.mapred.JobConf;
-
-
-
-/**
- * Singleton for accessing the parsed and merged system configuration.
- *
- * NOTE: parallel execution of multiple DML scripts (in the same JVM) with different configurations
- * would require changes/extensions of this class.
- */
-public class ConfigurationManager
-{
-
- private static DMLConfig _conf = null; //read systemml configuration
- private static JobConf _rJob = null; //cached job conf for read-only operations
-
- static{
- _rJob = new JobConf();
- }
-
-
- /**
- *
- * @param conf
- */
- public synchronized static void setConfig( DMLConfig conf )
- {
- _conf = conf;
- }
-
- /**
- *
- * @return
- */
- public synchronized static DMLConfig getConfig()
- {
- return _conf;
- }
-
- /**
- * Returns a cached JobConf object, intended for global use by all operations
- * with read-only access to job conf. This prevents to read the hadoop conf files
- * over and over again from classpath. However,
- *
- * @return
- */
- public static JobConf getCachedJobConf()
- {
- return _rJob;
- }
-
- /**
- *
- * @param job
- */
- public static void setCachedJobConf(JobConf job)
- {
- _rJob = job;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/conf/DMLConfig.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/conf/DMLConfig.java b/src/main/java/com/ibm/bi/dml/conf/DMLConfig.java
deleted file mode 100644
index 2b54f7b..0000000
--- a/src/main/java/com/ibm/bi/dml/conf/DMLConfig.java
+++ /dev/null
@@ -1,491 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.conf;
-
-import java.io.ByteArrayInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.StringWriter;
-import java.util.HashMap;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.transform.OutputKeys;
-import javax.xml.transform.Transformer;
-import javax.xml.transform.TransformerFactory;
-import javax.xml.transform.dom.DOMSource;
-import javax.xml.transform.stream.StreamResult;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-import org.w3c.dom.NodeList;
-import org.xml.sax.SAXException;
-
-import com.ibm.bi.dml.parser.ParseException;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.util.LocalFileUtils;
-
-
-public class DMLConfig
-{
-
- public static final String DEFAULT_SYSTEMML_CONFIG_FILEPATH = "./SystemML-config.xml";
-
- private static final Log LOG = LogFactory.getLog(DMLConfig.class.getName());
-
- // external names of configuration properties
- // (single point of change for all internal refs)
- public static final String LOCAL_TMP_DIR = "localtmpdir";
- public static final String SCRATCH_SPACE = "scratch";
- public static final String OPTIMIZATION_LEVEL = "optlevel";
- public static final String NUM_REDUCERS = "numreducers";
- public static final String JVM_REUSE = "jvmreuse";
- public static final String DEFAULT_BLOCK_SIZE = "defaultblocksize";
- public static final String YARN_APPMASTER = "dml.yarn.appmaster";
- public static final String YARN_APPMASTERMEM = "dml.yarn.appmaster.mem";
- public static final String YARN_MAPREDUCEMEM = "dml.yarn.mapreduce.mem";
- public static final String YARN_APPQUEUE = "dml.yarn.app.queue";
- public static final String CP_PARALLEL_MATRIXMULT = "cp.parallel.matrixmult";
- public static final String CP_PARALLEL_TEXTIO = "cp.parallel.textio";
-
- //obsolete nimble configuration (removed 06/24/2015)
- //public static final String NUM_MERGE_TASKS = "NumMergeTasks";
- //public static final String NUM_SOW_THREADS = "NumberOfSowThreads";
- //public static final String NUM_REAP_THREADS = "NumberOfReapThreads";
- //public static final String SOWER_WAIT_INTERVAL = "SowerWaitInterval";
- //public static final String REAPER_WAIT_INTERVAL = "ReaperWaitInterval";
- //public static final String NIMBLE_SCRATCH = "NimbleScratch";
-
- //internal config
- public static final String DEFAULT_SHARED_DIR_PERMISSION = "777"; //for local fs and DFS
- public static String LOCAL_MR_MODE_STAGING_DIR = null;
-
- //configuration default values
- private static HashMap<String, String> _defaultVals = null;
-
- private String config_file_name = null;
- private Element xml_root = null;
-
- static
- {
- _defaultVals = new HashMap<String, String>();
- _defaultVals.put(LOCAL_TMP_DIR, "/tmp/systemml" );
- _defaultVals.put(SCRATCH_SPACE, "scratch_space" );
- _defaultVals.put(OPTIMIZATION_LEVEL, "2" );
- _defaultVals.put(NUM_REDUCERS, "10" );
- _defaultVals.put(JVM_REUSE, "false" );
- _defaultVals.put(DEFAULT_BLOCK_SIZE, "1000" );
- _defaultVals.put(YARN_APPMASTER, "false" );
- _defaultVals.put(YARN_APPMASTERMEM, "2048" );
- _defaultVals.put(YARN_MAPREDUCEMEM, "-1" );
- _defaultVals.put(YARN_APPQUEUE, "default" );
- //_defaultVals.put(NUM_MERGE_TASKS, "4" );
- //_defaultVals.put(NUM_SOW_THREADS, "1" );
- //_defaultVals.put(NUM_REAP_THREADS, "1" );
- //_defaultVals.put(SOWER_WAIT_INTERVAL, "1000" );
- //_defaultVals.put(REAPER_WAIT_INTERVAL, "1000" );
- //_defaultVals.put(NIMBLE_SCRATCH, "nimbleoutput" );
- _defaultVals.put(CP_PARALLEL_MATRIXMULT, "true" );
- _defaultVals.put(CP_PARALLEL_TEXTIO, "true" );
- }
-
- public DMLConfig()
- {
-
- }
-
- /**
- *
- * @param fileName
- * @throws ParseException
- * @throws FileNotFoundException
- */
- public DMLConfig(String fileName)
- throws ParseException, FileNotFoundException
- {
- this( fileName, false );
- }
-
- /**
- *
- * @param fileName
- * @param silent
- * @throws ParseException
- * @throws FileNotFoundException
- */
- public DMLConfig(String fileName, boolean silent)
- throws ParseException, FileNotFoundException
- {
- config_file_name = fileName;
- try {
- parseConfig();
- } catch (FileNotFoundException fnfe) {
- LOCAL_MR_MODE_STAGING_DIR = getTextValue(LOCAL_TMP_DIR) + "/hadoop/mapred/staging";
- throw fnfe;
- } catch (Exception e){
- //log error, since signature of generated ParseException doesn't allow to pass it
- if( !silent )
- LOG.error("Failed to parse DML config file ",e);
- throw new ParseException("ERROR: error parsing DMLConfig file " + fileName);
- }
-
- LOCAL_MR_MODE_STAGING_DIR = getTextValue(LOCAL_TMP_DIR) + "/hadoop/mapred/staging";
- }
-
-
- public String getConfig_file_name()
- {
- return config_file_name;
- }
-
- public DMLConfig( Element root )
- {
- xml_root = root;
- }
-
- public void merge(DMLConfig otherConfig)
- throws ParseException
- {
- if (otherConfig == null)
- return;
-
- try {
- // for each element in otherConfig, either overwrite existing value OR add to defaultConfig
- NodeList otherConfigNodeList = otherConfig.xml_root.getChildNodes();
- if (otherConfigNodeList != null && otherConfigNodeList.getLength() > 0){
- for (int i=0; i<otherConfigNodeList.getLength(); i++){
- org.w3c.dom.Node optionalConfigNode = otherConfigNodeList.item(i);
-
- if (optionalConfigNode.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE){
-
- // try to find optional config node in default config node
- String paramName = optionalConfigNode.getNodeName();
- String paramValue = ((Element)optionalConfigNode).getFirstChild().getNodeValue();
-
- if (this.xml_root.getElementsByTagName(paramName) != null)
- LOG.info("Updating " + paramName + " with value " + paramValue);
- else
- LOG.info("Defining new attribute" + paramName + " with value " + paramValue);
- DMLConfig.setTextValue(this.xml_root, paramName, paramValue);
- }
-
- }
- } // end if (otherConfigNodeList != null && otherConfigNodeList.getLength() > 0){
- } catch (Exception e){
- LOG.error("Failed in merge default config file with optional config file",e);
- throw new ParseException("ERROR: error merging config file" + otherConfig.config_file_name + " with " + config_file_name);
- }
- }
-
- /**
- * Method to parse configuration
- * @throws ParserConfigurationException
- * @throws SAXException
- * @throws IOException
- */
- private void parseConfig () throws ParserConfigurationException, SAXException, IOException
- {
- DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
- factory.setIgnoringComments(true); //ignore XML comments
- DocumentBuilder builder = factory.newDocumentBuilder();
- Document domTree = null;
- if (config_file_name.startsWith("hdfs:") ||
- config_file_name.startsWith("gpfs:") ) // config file from DFS
- {
- if( !LocalFileUtils.validateExternalFilename(config_file_name, true) )
- throw new IOException("Invalid (non-trustworthy) hdfs config filename.");
- FileSystem DFS = FileSystem.get(ConfigurationManager.getCachedJobConf());
- Path configFilePath = new Path(config_file_name);
- domTree = builder.parse(DFS.open(configFilePath));
- }
- else // config from local file system
- {
- if( !LocalFileUtils.validateExternalFilename(config_file_name, false) )
- throw new IOException("Invalid (non-trustworthy) local config filename.");
- domTree = builder.parse(config_file_name);
- }
-
- xml_root = domTree.getDocumentElement();
- }
-
- /**
- * Method to get string value of a configuration parameter
- * Handles processing of configuration parameters
- * @param tagName the name of the DMLConfig parameter being retrieved
- * @return a string representation of the DMLConfig parameter value.
- */
- public String getTextValue(String tagName)
- {
- //get the actual value
- String retVal = (xml_root!=null)?getTextValue(xml_root,tagName):null;
-
- if (retVal == null)
- {
- if( _defaultVals.containsKey(tagName) )
- retVal = _defaultVals.get(tagName);
- else
- LOG.error("Error: requested dml configuration property '"+tagName+"' is invalid.");
- }
-
- return retVal;
- }
-
- public int getIntValue( String tagName )
- {
- return Integer.parseInt( getTextValue(tagName) );
- }
-
- public boolean getBooleanValue( String tagName )
- {
- return Boolean.parseBoolean( getTextValue(tagName) );
- }
-
-
- /**
- * Method to get the string value of an element identified by tag
- * @param ele
- * @param tagName
- * @return
- */
- private static String getTextValue(Element element, String tagName) {
- String textVal = null;
- NodeList list = element.getElementsByTagName(tagName);
- if (list != null && list.getLength() > 0) {
- Element elem = (Element) list.item(0);
- textVal = elem.getFirstChild().getNodeValue();
-
- }
- return textVal;
- }
-
- /**
- * Method to update the string value of an element identified by tagname
- * @param ele
- * @param tagName
- * @param newTextValue
- */
- private static void setTextValue(Element element, String tagName, String newTextValue) {
-
- NodeList list = element.getElementsByTagName(tagName);
- if (list != null && list.getLength() > 0) {
- Element elem = (Element) list.item(0);
- elem.getFirstChild().setNodeValue(newTextValue);
- }
- }
-
- /**
- *
- * @return
- * @throws DMLRuntimeException
- */
- public synchronized String serializeDMLConfig()
- throws DMLRuntimeException
- {
- String ret = null;
- try
- {
- Transformer transformer = TransformerFactory.newInstance().newTransformer();
- transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
- //transformer.setOutputProperty(OutputKeys.INDENT, "yes");
- StreamResult result = new StreamResult(new StringWriter());
- DOMSource source = new DOMSource(xml_root);
- transformer.transform(source, result);
- ret = result.getWriter().toString();
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException("Unable to serialize DML config.", ex);
- }
-
- return ret;
- }
-
- /**
- *
- * @param content
- * @return
- * @throws DMLRuntimeException
- */
- public static DMLConfig parseDMLConfig( String content )
- throws DMLRuntimeException
- {
- DMLConfig ret = null;
- try
- {
- //System.out.println(content);
- DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
- Document domTree = null;
- domTree = builder.parse( new ByteArrayInputStream(content.getBytes("utf-8")) );
- Element root = domTree.getDocumentElement();
- ret = new DMLConfig( root );
- }
- catch(Exception ex)
- {
- throw new DMLRuntimeException("Unable to parse DML config.", ex);
- }
-
- return ret;
- }
-
- /**
- *
- * @return
- * @throws ParseException
- * @throws FileNotFoundException
- */
- public static DMLConfig readAndMergeConfigurationFiles( String optConfig )
- throws ParseException, FileNotFoundException
- {
- // optional config specified overwrites/merge into the default config
- DMLConfig defaultConfig = null;
- DMLConfig optionalConfig = null;
-
- if( optConfig != null ) { // the optional config is specified
- try { // try to get the default config first
- defaultConfig = new DMLConfig(DEFAULT_SYSTEMML_CONFIG_FILEPATH, true);
- } catch (FileNotFoundException fnfe) { // it is OK to not have the default, but give a warning
- LOG.warn("No default SystemML config file (" + DEFAULT_SYSTEMML_CONFIG_FILEPATH + ") found");
- } catch (ParseException e) {
- defaultConfig = null;
- throw e;
- }
- try { // try to get the optional config next
- optionalConfig = new DMLConfig(optConfig, false);
- } catch (FileNotFoundException fnfe) {
- LOG.error("Config file " + optConfig + " not found");
- throw fnfe;
- } catch (ParseException e) {
- optionalConfig = null;
- throw e;
- }
- if (defaultConfig != null) {
- try {
- defaultConfig.merge(optionalConfig);
- }
- catch(ParseException e){
- defaultConfig = null;
- throw e;
- }
- }
- else {
- defaultConfig = optionalConfig;
- }
- }
- else { // the optional config is not specified
- try { // try to get the default config
- defaultConfig = new DMLConfig(DEFAULT_SYSTEMML_CONFIG_FILEPATH, false);
- } catch (FileNotFoundException fnfe) { // it is OK to not have the default, but give a warning
- LOG.warn("No default SystemML config file (" + DEFAULT_SYSTEMML_CONFIG_FILEPATH + ") found");
- LOG.warn("Using default settings in DMLConfig");
- DMLConfig dmlConfig = new DMLConfig();
- return dmlConfig;
- } catch (ParseException e) {
- defaultConfig = null;
- throw e;
- }
- }
-
- return defaultConfig;
- }
-
- /**
- *
- * @return
- */
- public String getConfigInfo()
- {
- String[] tmpConfig = new String[] {
- LOCAL_TMP_DIR,SCRATCH_SPACE,OPTIMIZATION_LEVEL,
- NUM_REDUCERS, DEFAULT_BLOCK_SIZE,
- YARN_APPMASTER, YARN_APPMASTERMEM, YARN_MAPREDUCEMEM,
- //NUM_MERGE_TASKS, NUM_SOW_THREADS,NUM_REAP_THREADS,
- //SOWER_WAIT_INTERVAL,REAPER_WAIT_INTERVAL,NIMBLE_SCRATCH
- CP_PARALLEL_MATRIXMULT, CP_PARALLEL_TEXTIO
- };
-
- StringBuilder sb = new StringBuilder();
- for( String tmp : tmpConfig )
- {
- sb.append("INFO: ");
- sb.append(tmp);
- sb.append(": ");
- sb.append(getTextValue(tmp));
- sb.append("\n");
- }
-
- return sb.toString();
- }
-
- /**
- *
- * @param amMem
- * @param mrMem
- */
- public void updateYarnMemorySettings(String amMem, String mrMem)
- {
- //app master memory
- NodeList list1 = xml_root.getElementsByTagName(YARN_APPMASTERMEM);
- if (list1 != null && list1.getLength() > 0) {
- Element elem = (Element) list1.item(0);
- elem.getFirstChild().setNodeValue(String.valueOf(amMem));
- }
-
- //mapreduce memory
- NodeList list2 = xml_root.getElementsByTagName(YARN_MAPREDUCEMEM);
- if (list2 != null && list2.getLength() > 0) {
- Element elem = (Element) list2.item(0);
- elem.getFirstChild().setNodeValue(String.valueOf(mrMem));
- }
- }
-
- /**
- *
- * @throws IOException
- */
- @SuppressWarnings("deprecation")
- public void makeQualifiedScratchSpacePath()
- throws IOException
- {
- NodeList list2 = xml_root.getElementsByTagName(SCRATCH_SPACE);
- if (list2 != null && list2.getLength() > 0) {
- Element elem = (Element) list2.item(0);
-
- FileSystem fs = FileSystem.get(ConfigurationManager.getCachedJobConf());
- String fname = elem.getFirstChild().getNodeValue();
- Path path = new Path(fname).makeQualified(fs);
-
- elem.getFirstChild().setNodeValue(path.toString());
- }
- }
-
- /**
- *
- * @param key
- * @return
- */
- public static String getDefaultTextValue( String key )
- {
- return _defaultVals.get( key );
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/debug/DMLBreakpointManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/debug/DMLBreakpointManager.java b/src/main/java/com/ibm/bi/dml/debug/DMLBreakpointManager.java
deleted file mode 100644
index 80b0447..0000000
--- a/src/main/java/com/ibm/bi/dml/debug/DMLBreakpointManager.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.debug;
-
-import java.util.TreeMap;
-
-import com.ibm.bi.dml.runtime.instructions.cp.BreakPointInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.BreakPointInstruction.BPINSTRUCTION_STATUS;
-
-
-/**
- * Class for managing breakpoints within DML compiler and debugger
- */
-public class DMLBreakpointManager {
-
- /** Map between DML script line numbers and breakpoint instructions */
- private static TreeMap<Integer, BreakPointInstruction> breakpoints = new TreeMap<Integer, BreakPointInstruction>();
-
-
- /**
- * Getter for DML breakpoints
- * @return List of breakpoints indexed by DML script line number
- */
- public static TreeMap<Integer, BreakPointInstruction> getBreakpoints()
- {
- if (breakpoints.size() > 0)
- return breakpoints;
- return null;
- }
-
- /**
- * Returns size of active DML breakpoints
- * @return size Size of active breakpoints
- */
- public static int getBreakpointsSize()
- {
- int size = 0;
- for (Integer lineNumber : breakpoints.keySet()) {
- if (breakpoints.get(lineNumber).getBPInstructionStatus() != BPINSTRUCTION_STATUS.INVISIBLE)
- size++;
- }
- return size;
- }
-
- /**
- * Returns breakpoint instruction at a particular line number (if any)
- * @param lineNumber Location of breakpoint
- * @return Breakpoint instruction at indicated line number (if any)
- */
- public static BreakPointInstruction getBreakpoint(int lineNumber) {
- if (!breakpoints.containsKey(lineNumber))
- return null;
- return breakpoints.get(lineNumber);
- }
-
- /**
- * Returns breakpoint instruction with given breakpoint id
- * @param location Breakpoint id
- * @return Breakpoint instruction at indicated id
- */
- public static BreakPointInstruction getBreakpointAtIndex(int location) {
- int index = 1;
- for (Integer lineNumber : breakpoints.keySet()) {
- if (index++ == location) {
- return breakpoints.get(lineNumber);
- }
- }
- return null;
- }
-
- /**
- * Returns breakpoint line number with given breakpoint id
- * @param location Breakpoint id
- * @return Breakpoint instruction line number (-1 if not found)
- */
- public static int getBreakpointLineNumber(int location) {
- int index = 1;
- for (Integer lineNumber : breakpoints.keySet()) {
- if (index++ == location) {
- return lineNumber;
- }
- }
- return -1;
- }
-
- /**
- * Returns breakpoint identifier with given line number
- * @param Line number Location of breakpoint in DML script
- * @return bpID Breakpoint id within all breakpoints (-1 if not found)
- */
- public static int getBreakpointID(int lineNum) {
- int bpID=1;
- for (Integer lineNumber : breakpoints.keySet()) {
- if (lineNum == lineNumber) {
- return bpID;
- }
- bpID++;
- }
- return -1;
- }
-
- /**
- * Insert a breakpoint instruction into list of existing breakpoints
- * @param lineNumber Location for inserting breakpoint
- */
- public static void insertBreakpoint (BreakPointInstruction breakpoint, int lineNumber) {
- if (breakpoints.containsKey(lineNumber)) {
- if (breakpoints.get(lineNumber).getBPInstructionStatus() != BPINSTRUCTION_STATUS.INVISIBLE)
- System.out.format("Breakpoint updated at %s, line, %d.\n", breakpoint.getBPInstructionLocation(), lineNumber);
- else
- System.out.format("Breakpoint added at %s, line %d.\n", breakpoint.getBPInstructionLocation(), lineNumber);
- breakpoints.put(lineNumber, breakpoint);
- }
- }
-
- /**
- * Insert a breakpoint instruction into list of breakpoints
- * @param lineNumber Location for inserting breakpoint
- */
- public static void insertBreakpoint (int lineNumber) {
- if (breakpoints.containsKey(lineNumber)) {
- breakpoints.get(lineNumber).setBPInstructionStatus(BPINSTRUCTION_STATUS.INVISIBLE);
- }
- else {
- breakpoints.put(lineNumber, new BreakPointInstruction(BPINSTRUCTION_STATUS.INVISIBLE));
- }
- }
-
- /**
- * Updates breakpoint status for a given breakpoint id
- * @param location Breakpoint identifier
- * @param status Current breakpoint status
- */
- public static void updateBreakpoint(int lineNumber, BPINSTRUCTION_STATUS status) {
- if (breakpoints.containsKey(lineNumber)) {
- breakpoints.get(lineNumber).setBPInstructionStatus(status);
- System.out.format("Breakpoint updated at %s, line %d.\n", breakpoints.get(lineNumber).getBPInstructionLocation(), lineNumber);
- }
- }
-
- /**
- * Updates breakpoint status for a given breakpoint id
- * @param location Breakpoint identifier
- * @param status Current breakpoint status
- */
- public static void updateBreakpointID(int location, BPINSTRUCTION_STATUS status) {
- int lineNumber = getBreakpointLineNumber(location);
- if (lineNumber != -1) {
- breakpoints.get(lineNumber).setBPInstructionStatus(status);
- System.out.format("Breakpoint updated at %s, line %d.\n", breakpoints.get(lineNumber).getBPInstructionLocation(), lineNumber);
- }
- }
-
- /**
- * Removes breakpoint instruction at given line number
- * @param lineNumber Location for inserting breakpoint
- * @param status Current breakpoint status
- */
- public static void removeBreakpoint(int lineNumber, BPINSTRUCTION_STATUS status) {
- if (breakpoints.containsKey(lineNumber)) {
- breakpoints.get(lineNumber).setBPInstructionStatus(status);
- System.out.format("Breakpoint deleted at %s, line %d.\n", breakpoints.get(lineNumber).getBPInstructionLocation(), lineNumber);
- }
- }
-
- /**
- * Removes breakpoint instruction at given location
- * @param location Breakpoint instruction id
- * @param status Current breakpoint status
- */
- public static void removeBreakpointIndex(int location, BPINSTRUCTION_STATUS status) {
- int lineNumber = getBreakpointLineNumber(location);
- if (lineNumber != -1)
- breakpoints.get(lineNumber).setBPInstructionStatus(status);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/debug/DMLDebugger.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/debug/DMLDebugger.java b/src/main/java/com/ibm/bi/dml/debug/DMLDebugger.java
deleted file mode 100644
index 33d8116..0000000
--- a/src/main/java/com/ibm/bi/dml/debug/DMLDebugger.java
+++ /dev/null
@@ -1,609 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.debug;
-
-import java.io.PrintStream;
-import java.util.HashMap;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.lang.math.IntRange;
-
-import com.ibm.bi.dml.debug.DMLDebuggerFunctions;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext;
-import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContextFactory;
-import com.ibm.bi.dml.runtime.instructions.cp.BreakPointInstruction.BPINSTRUCTION_STATUS;
-
-/**
- * This class implements a debugger control module for DML scripts.
- * Note: ONLY USED FOR DEBUGGING PURPOSES
- */
-public class DMLDebugger
-{
-
-
- // This will be supported in subsquent release. turning off the -debug 'optimize' feature for current release.
- //public static boolean ENABLE_DEBUG_OPTIMIZER = false; //default debug mode
- //public static boolean ENABLE_SERVER_SIDE_DEBUG_MODE = false; //default debug mode
-
- //public static final String DEBUGGER_SYSTEMML_CONFIG_FILEPATH = "./DebuggerSystemML-config.xml";
-
-
- private DMLDebuggerProgramInfo dbprog; //parsed and compiled DML script w/ hops, lops and runtime program
- public DMLDebuggerInterface debuggerUI; //debugger command line interface
- private DMLDebuggerFunctions dbFunctions; //debugger functions interface
- private CommandLine cmd; //debugger function command
-
- //support for obtaining STDOUT/STDERR streams of DML program running in debug mode
- private PrintStream originalOut = null;
- private PrintStream originalErr = null;
-
- private String dmlScriptStr; //DML script contents (including new lines)
- HashMap<String,String> argVals; //key-value pairs defining arguments of DML script
- ExecutionContext preEC = null;
- ExecutionContext currEC = null;
- String [] lines;
- volatile boolean quit=false;
-
- /**
- * Constructor for DML debugger CLI
- */
- public DMLDebugger(DMLDebuggerProgramInfo p, String dmlScript, HashMap<String,String> args)
- {
- dbprog = p;
- dmlScriptStr = dmlScript;
- lines = dmlScriptStr.split("\n");
- argVals = args;
- debuggerUI = new DMLDebuggerInterface();
- dbFunctions = new DMLDebuggerFunctions();
- preEC = ExecutionContextFactory.createContext(dbprog.rtprog);
- setupDMLRuntime();
- }
-
- /**
- * Sets up DML runtime with DML script and instructions information
- */
- private void setupDMLRuntime()
- {
- dbprog.setDMLInstMap();
- preEC.getDebugState().setDMLScript(lines);
- }
-
- /**
- * Sets STDOUT stream of a DML program running in debug mode
- */
- @SuppressWarnings("unused")
- private void setStdOut()
- {
- originalOut = System.out;
- System.setOut(originalOut);
- }
-
- /**
- * Gets STDOUT stream of a DML program running in debug mode
- * @return STDOUT stream of DML program
- */
- @SuppressWarnings("unused")
- private PrintStream getStdOut()
- {
- System.out.flush();
- return originalOut;
- }
-
- /**
- * Sets STDERR stream of a DML program running in debug mode
- */
- @SuppressWarnings("unused")
- private void setStdErr()
- {
- originalErr = System.err;
- System.setOut(originalErr);
- }
-
- /**
- * Gets STDERR stream of a DML program running in debug mode
- * @return STDERR stream of DML program
- */
- @SuppressWarnings("unused")
- private PrintStream getStdErr()
- {
- System.err.flush();
- return originalErr;
- }
-
- /**
- * Get debug function command from debugger CLI
- * @throws DMLDebuggerException
- */
- private void getCommand() throws DMLDebuggerException
- {
- cmd = debuggerUI.getDebuggerCommand();
- }
-
- /**
- * Class for running the DML runtime as a thread
- */
- Runnable DMLRuntime = new Runnable()
- {
- public void run() {
- try {
- // System.out.println("Starting DML script ...");
- dbprog.rtprog.execute(currEC);
- // System.out.println("DML script has finished execution.");
- synchronized(DMLDebugger.class) {
- quit = true;
- }
- }
- catch (Exception e) {
- System.err.println("Exception raised by DML runtime:" + e);
- }
- }
- };
-
- /**
- * Controls the communication between debugger CLI and DML runtime.
- */
- @SuppressWarnings("deprecation")
- public synchronized void runSystemMLDebugger()
- {
- debuggerUI.setOptions();
- debuggerUI.getDebuggerCLI();
- Thread runtime = new Thread(DMLRuntime);
- boolean isRuntimeInstruction = false;
-
- while (!quit) {
- try {
- //get debugger function from CLI
- getCommand();
- if(cmd != null) {
- isRuntimeInstruction = false;
- //check for help
- if(cmd.hasOption("h")) {
- debuggerUI.getDebuggerCLI();
- }
- //check for exit
- else if (cmd.hasOption("q")) {
- synchronized(DMLDebugger.class) {
- quit = true;
- }
- runtime.stop();
- }
- else if (cmd.hasOption("r")) {
- if (currEC != null) {
- System.out.println("Runtime has already started. Try \"s\" to go to next line, or \"c\" to continue running your DML script.");
- }
- else {
- currEC = preEC;
- runtime.start();
- isRuntimeInstruction = true;
- }
- }
- else if (cmd.hasOption("c")) {
- if (currEC == null)
- System.out.println("Runtime has not been started. Try \"r\" to start DML runtime execution.");
- else if (!runtime.isAlive()) {
- System.err.println("Invalid debug state.");
- //System.out.println("Runtime terminated. Try \"-c\" to recompile followed by \"r\" to restart DML runtime execution.");
- }
- else {
- System.out.println("Resuming DML script execution ...");
- preEC.getDebugState().setCommand(null);
- runtime.resume();
- isRuntimeInstruction = true;
- }
- }
- else if (cmd.hasOption("si")) {
- if (!runtime.isAlive()) {
- currEC = preEC;
- runtime.start();
- isRuntimeInstruction = true;
- // System.out.println("Runtime must be started before single stepping can be enabled. Try \"r\" to start DML runtime execution.");
- }
- //else {
- preEC.getDebugState().setCommand("step_instruction");
- runtime.resume();
- isRuntimeInstruction = true;
- //}
- }
- else if (cmd.hasOption("s")) {
- if (!runtime.isAlive()) {
- currEC = preEC;
- runtime.start();
- isRuntimeInstruction = true;
- //System.out.println("Runtime must be started before step over can be enabled. Try \"r\" to start DML runtime execution.");
- }
- //else {
- preEC.getDebugState().setCommand("step_line");
- runtime.resume();
- isRuntimeInstruction = true;
- //}
- }
-// else if (cmd.hasOption("step_return")) {
-// if (!runtime.isAlive()) {
-// System.out.println("Runtime must be started before step return can be enabled. Try \"r\" to start DML runtime execution.");
-// }
-// else {
-// String fname = dbFunctions.getValue(cmd.getOptionValues("step_return"));
-// dbprog.rtprog.setCommand("step return");
-// if (fname != null) {
-// dbprog.rtprog.setCommandArg(fname);
-// }
-// runtime.resume();
-// isRuntimeInstruction = true;
-// }
-// }
- else if (cmd.hasOption("b")) {
- int lineNumber = dbFunctions.getValue(cmd.getOptionValues("b"), lines.length);
- if (lineNumber > 0) {
- if (DMLBreakpointManager.getBreakpoint(lineNumber) == null)
- System.out.println("Sorry, a breakpoint cannot be inserted at line " + lineNumber + ". Please try a different line number.");
- else {
- if (DMLBreakpointManager.getBreakpoint(lineNumber).getBPInstructionStatus() != BPINSTRUCTION_STATUS.INVISIBLE) {
- System.out.format("Breakpoint at line %d already exists.\n", lineNumber);
- }
- else {
- dbprog.accessBreakpoint(lineNumber, 0, BPINSTRUCTION_STATUS.ENABLED);
- }
- }
- }
- }
- else if (cmd.hasOption("d")) {
- int lineNumber = dbFunctions.getValue(cmd.getOptionValues("d"), lines.length);
- if (lineNumber > 0 && DMLBreakpointManager.getBreakpoint(lineNumber) != null &&
- DMLBreakpointManager.getBreakpoint(lineNumber).getBPInstructionStatus() != BPINSTRUCTION_STATUS.INVISIBLE) {
- dbprog.accessBreakpoint(lineNumber, 1, BPINSTRUCTION_STATUS.INVISIBLE);
- //dbprog.accessBreakpoint(lineNumber, 1, BPINSTRUCTION_STATUS.DISABLED);
- }
- else {
- System.out.println("Sorry, a breakpoint cannot be deleted at line " + lineNumber + ". Please try a different line number.");
- }
-
- }
- else if (cmd.hasOption("i")) {
- String [] infoOptions = cmd.getOptionValues("i");
- if(infoOptions == null || infoOptions.length == 0) {
- System.err.println("The command \"info\" requires option. Try \"info break\" or \"info frame\".");
- }
- else if(infoOptions[0].trim().compareTo("break") == 0) {
- dbFunctions.listBreakpoints(DMLBreakpointManager.getBreakpoints());
- }
- else if(infoOptions[0].trim().compareTo("frame") == 0) {
- if (!runtime.isAlive())
- System.err.println("Runtime has not been started. Try \"r\" or \"s\" to start DML runtime execution.");
- else
- dbFunctions.printCallStack(currEC.getDebugState().getCurrentFrame(), currEC.getDebugState().getCallStack());
- }
- else {
- System.err.println("Invalid option for command \"info\". Try \"info break\" or \"info frame\".");
- }
- }
- else if (cmd.hasOption("p")) {
- String [] pOptions = cmd.getOptionValues("p");
- if(pOptions == null || pOptions.length != 1) {
- System.err.println("Incorrect options for command \"print\"");
- }
- else {
- String varName = pOptions[0].trim();
- if (runtime.isAlive()) {
- if(varName.contains("[")) {
- // matrix with index: can be cell or column or row
- try {
- //System.out.println("" + varName);
- String variableNameWithoutIndices = varName.split("\\[")[0].trim();
- //System.out.println("" + variableNameWithoutIndices);
- String indexString = (varName.split("\\[")[1].trim()).split("\\]")[0].trim();
- //System.out.println(">>" + indexString + "<<");
- String rowIndexStr = "";
- String colIndexStr = "";
- if(indexString.startsWith(",")) {
-
- colIndexStr = indexString.split(",")[1].trim();
- }
- else if(indexString.endsWith(",")) {
- rowIndexStr = indexString.split(",")[0].trim();
- }
- else {
- rowIndexStr = indexString.split(",")[0].trim();
- colIndexStr = indexString.split(",")[1].trim();
- }
- int rowIndex = -1;
- int colIndex = -1;
- if(rowIndexStr.compareTo("") != 0) {
- rowIndex = Integer.parseInt(rowIndexStr);
- }
- if(colIndexStr.compareTo("") != 0) {
- colIndex = Integer.parseInt(colIndexStr);
- }
- //System.out.println("" + rowIndex + " " + colIndex);
- dbFunctions.print(currEC.getDebugState().getVariables(), variableNameWithoutIndices, "value", rowIndex, colIndex);
- }
- catch(Exception indicesException) {
- System.err.println("Incorrect fomat for \"p\". If you are trying to print matrix variable M, you can use M[1,] or M[,1] or M[1,1] (without spaces).");
- }
- }
- else {
- // Print entire matrix
- dbFunctions.print(currEC.getDebugState().getVariables(), varName, "value", -1, -1);
- }
- }
- else
- System.err.println("Runtime has not been started. Try \"r\" or \"s\" to start DML runtime execution.");
- }
- }
- else if (cmd.hasOption("whatis")) {
- String [] pOptions = cmd.getOptionValues("whatis");
- if(pOptions == null || pOptions.length != 1) {
- System.err.println("Incorrect options for command \"whatis\"");
- }
- else {
- String varName = pOptions[0].trim();
- dbFunctions.print(currEC.getDebugState().getVariables(), varName, "metadata", -1, -1);
- }
- }
- else if (cmd.hasOption("set")) {
- String [] pOptions = cmd.getOptionValues("set");
- if(pOptions == null || pOptions.length != 2) {
- System.err.println("Incorrect options for command \"set\"");
- }
- else {
- try {
- if(pOptions[0].contains("[")) {
- String [] paramsToSetMatrix = new String[4];
- paramsToSetMatrix[0] = pOptions[0].split("\\[")[0].trim();
- String indexString = (pOptions[0].split("\\[")[1].trim()).split("\\]")[0].trim();
- paramsToSetMatrix[1] = indexString.split(",")[0].trim();
- paramsToSetMatrix[2] = indexString.split(",")[1].trim();
- paramsToSetMatrix[3] = pOptions[1].trim();
- dbFunctions.setMatrixCell(currEC.getDebugState().getVariables(), paramsToSetMatrix);
- }
- else {
- dbFunctions.setScalarValue(currEC.getDebugState().getVariables(), pOptions);
- }
- }
- catch(Exception exception1) {
- System.out.println("Only scalar variable or a matrix cell available in current frame can be set in current version.");
- }
- }
- }
- else if (cmd.hasOption("l")) {
-
- String [] pOptions = cmd.getOptionValues("l");
- String [] argsForRange = new String[2];
- int currentPC = 1;
-
- if(runtime.isAlive()) {
- currentPC = currEC.getDebugState().getPC().getLineNumber();
- }
-
- IntRange range = null;
- if(pOptions == null) {
- // Print first 10 lines
- range = new IntRange(currentPC, Math.min(lines.length, currentPC+10));
- }
- else if(pOptions.length == 1 && pOptions[0].trim().toLowerCase().compareTo("all") == 0) {
- // Print entire program
- range = new IntRange(1, lines.length);
- }
- else if(pOptions.length == 2 && pOptions[0].trim().toLowerCase().compareTo("next") == 0) {
- int numLines = 10;
- try {
- numLines = Integer.parseInt(pOptions[1]);
- }
- catch(Exception e1) {}
-
- argsForRange[0] = "" + currentPC;
- argsForRange[1] = "" + Math.min(lines.length, numLines + currentPC);
- range = dbFunctions.getRange(argsForRange, lines.length);
-
- }
- else if(pOptions.length == 2 && pOptions[0].trim().toLowerCase().compareTo("prev") == 0) {
- int numLines = 10;
- try {
- numLines = Integer.parseInt(pOptions[1]);
- }
- catch(Exception e1) {}
-
- argsForRange[0] = "" + Math.max(1, currentPC - numLines);
- argsForRange[1] = "" + currentPC;
- range = dbFunctions.getRange(argsForRange, lines.length);
- }
-
-
- if(range == null) {
- System.err.println("Incorrect usage of command \"l\". Try \"l\" or \"l all\" or \"l next 5\" or \"l prev 5\".");
- }
- else {
- if (range.getMinimumInteger() > 0) {
- dbFunctions.printLines(lines, range);
- }
- else {
- System.err.println("Sorry no lines that can be printed. Try \"l\" or \"l all\" or \"l next 5\" or \"l prev 5\".");
- }
- }
-
-
-
- // Old code:
- // IntRange range = dbFunctions.getRange(cmd.getOptionValues("p"), lines.length);
- //if (range.getMinimumInteger() > 0) {
- // dbFunctions.printLines(lines, range);
- // }
- }
- else if (cmd.hasOption("li")) {
-
- String [] pOptions = cmd.getOptionValues("li");
- String [] argsForRange = new String[2];
- int currentPC = 1;
-
- if(runtime.isAlive()) {
- currentPC = currEC.getDebugState().getPC().getLineNumber();
- }
-
- IntRange range = null;
- if(pOptions == null) {
- // Print first 10 lines
- range = new IntRange(currentPC, Math.min(lines.length, currentPC+10));
- }
- else if(pOptions.length == 1 && pOptions[0].trim().toLowerCase().compareTo("all") == 0) {
- // Print entire program
- range = new IntRange(1, lines.length);
- }
- else if(pOptions.length == 2 && pOptions[0].trim().toLowerCase().compareTo("next") == 0) {
- int numLines = 10;
- try {
- numLines = Integer.parseInt(pOptions[1]);
- }
- catch(Exception e1) {}
-
- argsForRange[0] = "" + currentPC;
- argsForRange[1] = "" + Math.min(lines.length, numLines + currentPC);
- range = dbFunctions.getRange(argsForRange, lines.length);
-
- }
- else if(pOptions.length == 2 && pOptions[0].trim().toLowerCase().compareTo("prev") == 0) {
- int numLines = 10;
- try {
- numLines = Integer.parseInt(pOptions[1]);
- }
- catch(Exception e1) {}
-
- argsForRange[0] = "" + Math.max(1, currentPC - numLines);
- argsForRange[1] = "" + currentPC;
- range = dbFunctions.getRange(argsForRange, lines.length);
- }
-
-
- if(range == null) {
- System.err.println("Incorrect usage of command \"li\". Try \"li\" or \"li all\" or \"li next 5\" or \"li prev 5\".");
- }
- else {
- if (range.getMinimumInteger() > 0) {
- dbFunctions.printInstructions(lines, dbprog.getDMLInstMap(), range, false);
- }
- else {
- System.err.println("Sorry no lines that can be printed. Try \"li\" or \"li all\" or \"li next 5\" or \"li prev 5\".");
- }
- }
-
-
-
- // Old code:
- // IntRange range = dbFunctions.getRange(cmd.getOptionValues("p"), lines.length);
- //if (range.getMinimumInteger() > 0) {
- // dbFunctions.printLines(lines, range);
- // }
- }
- else if (cmd.hasOption("set_scalar")) {
- if (!runtime.isAlive())
- System.err.println("Runtime has not been started. Try \"r\" to start DML runtime execution.");
- else
- dbFunctions.setScalarValue(currEC.getDebugState().getVariables(), cmd.getOptionValues("set_scalar"));
- }
- else if (cmd.hasOption("m")) {
- String varname = dbFunctions.getValue(cmd.getOptionValues("m"));
- if (runtime.isAlive())
- dbFunctions.printMatrixVariable(currEC.getDebugState().getVariables(), varname);
- else
- System.err.println("Runtime has not been started. Try \"r\" to start DML runtime execution.");
- }
- else if (cmd.hasOption("x")) {
- if (!runtime.isAlive())
- System.err.println("Runtime has not been started. Try \"r\" to start DML runtime execution.");
- else {
- dbFunctions.printMatrixCell(currEC.getDebugState().getVariables(), cmd.getOptionValues("x"));
- }
- }
- else if (cmd.hasOption("set_cell")) {
- if (!runtime.isAlive())
- System.err.println("Runtime has not been started. Try \"r\" to start DML runtime execution.");
- else {
- dbFunctions.setMatrixCell(currEC.getDebugState().getVariables(), cmd.getOptionValues("set_cell"));
- }
- }
- else {
- System.err.println("Undefined command. Try \"help\".");
- }
- //block until runtime suspends execution or terminates
- //while(runtime.isAlive() && !currEC.getProgram().isStopped()) {
- wait(300); // To avoid race condition between submitting job and
- //System.out.println(">> Before while");
- while(isRuntimeInstruction && !currEC.getDebugState().canAcceptNextCommand()) {
- if(quit){
- break;
- }
- else {
- wait(300); //wait
- }
- }
- //System.out.println(">> After while");
- }
- wait(300);
- } catch (Exception e) {
- System.err.println("Error processing debugger command. Try \"help\".");
- }
- }
- }
-
-
-// Since the recompile option is disabled in the debugger to make the interface much cleaner
-// /**
-// * compile: Compile DML script and generate hops, lops and runtime program for debugger.
-// * @param dmlScriptStr DML script contents (including new lines)
-// * @param argVals Key-value pairs defining arguments of DML script
-// * @throws ParseException
-// * @throws IOException
-// * @throws DMLRuntimeException
-// * @throws LanguageException
-// * @throws HopsException
-// * @throws LopsException
-// * @throws DMLUnsupportedOperationException
-// * @throws Exception
-// */
-// private void recompile(String dmlScriptStr, HashMap<String,String> argVals)
-// throws ParseException, IOException, DMLRuntimeException, LanguageException, HopsException, LopsException, DMLUnsupportedOperationException, Exception
-// {
-// dbprog = new DMLDebuggerProgramInfo();
-//
-// //Step 1: parse dml script
-// if(DMLScript.ENABLE_PYTHON_PARSING) {
-// PyDMLParserWrapper parser = new PyDMLParserWrapper();
-// dbprog.prog = parser.parse(DMLScript.DML_FILE_PATH_ANTLR_PARSER, dmlScriptStr, argVals);
-// }
-// else {
-// DMLParserWrapper parser = new DMLParserWrapper();
-// dbprog.prog = parser.parse(DMLScript.DML_FILE_PATH_ANTLR_PARSER, dmlScriptStr, argVals);
-// }
-//
-// //Step 3: construct HOP DAGs (incl LVA and validate)
-// dbprog.dmlt = new DMLTranslator(dbprog.prog);
-// dbprog.dmlt.liveVariableAnalysis(dbprog.prog);
-// dbprog.dmlt.validateParseTree(dbprog.prog);
-// dbprog.dmlt.constructHops(dbprog.prog);
-//
-// //Step 4: rewrite HOP DAGs (incl IPA and memory estimates)
-// dbprog.dmlt.rewriteHopsDAG(dbprog.prog);
-//
-// //Step 5: construct LOP DAGs
-// dbprog.dmlt.constructLops(dbprog.prog);
-//
-// //Step 6: generate runtime program
-// dbprog.rtprog = dbprog.prog.getRuntimeProgram(dbprog.conf);
-//
-// //Set debug mode flag
-// setupDMLRuntime();
-// }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/debug/DMLDebuggerException.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/debug/DMLDebuggerException.java b/src/main/java/com/ibm/bi/dml/debug/DMLDebuggerException.java
deleted file mode 100644
index 132a5c7..0000000
--- a/src/main/java/com/ibm/bi/dml/debug/DMLDebuggerException.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.debug;
-
-import com.ibm.bi.dml.api.DMLException;
-
-/**
- * This exception should be thrown to flag debugger errors.
- */
-public class DMLDebuggerException extends DMLException
-{
-
- private static final long serialVersionUID = 1L;
-
- public DMLDebuggerException(String string) {
- super(string);
- }
-
- public DMLDebuggerException(Exception e) {
- super(e);
- }
-
- public DMLDebuggerException(String string, Exception ex){
- super(string,ex);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/276d9257/src/main/java/com/ibm/bi/dml/debug/DMLDebuggerFunctions.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/ibm/bi/dml/debug/DMLDebuggerFunctions.java b/src/main/java/com/ibm/bi/dml/debug/DMLDebuggerFunctions.java
deleted file mode 100644
index 94ce010..0000000
--- a/src/main/java/com/ibm/bi/dml/debug/DMLDebuggerFunctions.java
+++ /dev/null
@@ -1,678 +0,0 @@
-/**
- * (C) Copyright IBM Corp. 2010, 2015
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package com.ibm.bi.dml.debug;
-
-import java.util.ArrayList;
-import java.util.Map.Entry;
-import java.util.Stack;
-import java.util.TreeMap;
-
-import org.apache.commons.lang.math.IntRange;
-
-import com.ibm.bi.dml.hops.OptimizerUtils;
-import com.ibm.bi.dml.lops.Lop;
-import com.ibm.bi.dml.parser.Expression.DataType;
-import com.ibm.bi.dml.runtime.DMLRuntimeException;
-import com.ibm.bi.dml.runtime.controlprogram.LocalVariableMap;
-import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
-import com.ibm.bi.dml.runtime.instructions.Instruction;
-import com.ibm.bi.dml.runtime.instructions.MRJobInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.BooleanObject;
-import com.ibm.bi.dml.runtime.instructions.cp.BreakPointInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.CPInstruction;
-import com.ibm.bi.dml.runtime.instructions.cp.Data;
-import com.ibm.bi.dml.runtime.instructions.cp.DoubleObject;
-import com.ibm.bi.dml.runtime.instructions.cp.IntObject;
-import com.ibm.bi.dml.runtime.instructions.cp.ScalarObject;
-import com.ibm.bi.dml.runtime.instructions.cp.StringObject;
-import com.ibm.bi.dml.runtime.instructions.cp.BreakPointInstruction.BPINSTRUCTION_STATUS;
-import com.ibm.bi.dml.runtime.matrix.data.MatrixBlock;
-
-public class DMLDebuggerFunctions {
-
- //Maximum values of rows and columns when displaying a DML matrix/vector
- public static final int DISPLAY_MAX_ROWS = 30;
- public static final int DISPLAY_MAX_COLUMNS = 8;
-
- /////////////////////////////////////////////
- // public interface for debugger functions //
- /////////////////////////////////////////////
-
- /**
- * Print all breakpoints along with current status (i.e. enabled or disabled)
- * @param breakpoints Contains all existing breakpoints
- */
- public void listBreakpoints(TreeMap<Integer, BreakPointInstruction> breakpoints)
- {
- //Display all breakpoints
- if (breakpoints == null)
- {
- System.out.println("No breakpoints are set for this program.");
- return;
- }
- int currBreakpoint = 1; //active breakpoint ids
- int numVisibleBreakpoints = 0;
- for (Entry<Integer, BreakPointInstruction> e : breakpoints.entrySet() )
- {
- Integer lineNumber = e.getKey();
- BreakPointInstruction inst = e.getValue();
-
- if (inst.getBPInstructionStatus() == BPINSTRUCTION_STATUS.ENABLED) {
- System.out.format("Breakpoint %2d, at line %4d (%s)\n", currBreakpoint++, lineNumber, "enabled");
- numVisibleBreakpoints++;
- }
- else if (inst.getBPInstructionStatus() == BPINSTRUCTION_STATUS.DISABLED) {
- System.out.format("Breakpoint %2d, at line %4d (%s)\n", currBreakpoint++, lineNumber, "disabled");
- numVisibleBreakpoints++;
- }
- }
-
- if(numVisibleBreakpoints == 0) {
- System.out.println("No breakpoints are set for this program.");
- }
- }
-
- /**
- * Print range of DML program lines
- * @param lines DML script lines of code
- * @param range Range of lines of DML code to be displayed
- */
- public void printLines(String [] lines, IntRange range)
- {
- //Display all lines of DML script
- for (int lineNumber=range.getMinimumInteger() ; lineNumber<=range.getMaximumInteger() ; lineNumber++)
- System.out.format("line %4d: %s\n", lineNumber, lines[lineNumber-1]);
- }
-
- /**
- * Print range of DML program lines interspersed with corresponding runtime instructions
- * @param lines DML script lines of code
- * @param DMLInstMap Mapping between source code line number and corresponding runtime instruction(s)
- * @param range Range of lines of DML code to be displayed
- * @param debug Flag for displaying instructions in debugger test integration
- */
- public void printInstructions(String [] lines, TreeMap<Integer, ArrayList<Instruction>> DMLInstMap, IntRange range, boolean debug)
- {
- //Display instructions with corresponding DML line numbers
- for (int lineNumber=range.getMinimumInteger() ; lineNumber<=range.getMaximumInteger() ; lineNumber++)
- {
- System.out.format("line %4d: %s\n", lineNumber, lines[lineNumber-1]);
- if (DMLInstMap.get(lineNumber) != null)
- {
- for (Instruction currInst : DMLInstMap.get(lineNumber))
- {
- if (currInst instanceof CPInstruction) {
- if (!debug)
- System.out.format("\t\t id %4d: %s\n", currInst.getInstID(), prepareInstruction(currInst.toString()));
- else {
- String [] instStr = prepareInstruction(currInst.toString()).split(" ");
- System.out.format("\t\t id %4d: %s %s\n", currInst.getInstID(), instStr[0], instStr[1]);
- }
- }
- else if (currInst instanceof MRJobInstruction)
- {
- MRJobInstruction currMRInst = (MRJobInstruction) currInst;
- System.out.format("\t\t id %4d: %s\n", currInst.getInstID(), prepareInstruction(currMRInst.getMRString(debug)));
- }
- else if (currInst instanceof BreakPointInstruction) {
- BreakPointInstruction currBPInst = (BreakPointInstruction) currInst;
- System.out.format("\t\t id %4d: %s\n", currInst.getInstID(), currBPInst.toString());
- }
- }
- }
- }
- }
-
- /**
- * Print range of program runtime instructions
- * @param DMLInstMap Mapping between source code line number and corresponding runtime instruction(s)
- * @param range Range of lines of DML code to be displayed
- */
- public void printRuntimeInstructions(TreeMap<Integer, ArrayList<Instruction>> DMLInstMap, IntRange range)
- {
- //Display instructions
- for (int lineNumber=range.getMinimumInteger() ; lineNumber<=range.getMaximumInteger() ; lineNumber++)
- {
- if (DMLInstMap.get(lineNumber) != null)
- {
- for (Instruction currInst : DMLInstMap.get(lineNumber))
- {
- if (currInst instanceof CPInstruction)
- System.out.format("\t\t id %4d: %s\n", currInst.getInstID(), prepareInstruction(currInst.toString()));
- else if (currInst instanceof MRJobInstruction)
- {
- MRJobInstruction currMRInst = (MRJobInstruction) currInst;
- System.out.format("\t\t id %4d: %s\n", currInst.getInstID(), prepareInstruction(currMRInst.getMRString(false)));
- }
- else if (currInst instanceof BreakPointInstruction) {
- BreakPointInstruction currBPInst = (BreakPointInstruction) currInst;
- System.out.format("\t\t id %4d: %s\n", currInst.getInstID(), currBPInst.toString());
- }
- }
- }
- }
- }
-
- /**
- * Print current program counter
- * @param pc Current stack frame program counter
- */
- public void printPC(DMLProgramCounter pc)
- {
- //Display program counter
- if (pc != null)
- System.out.println(" Current program counter at " + pc.toString());
- else
- System.out.println("DML runtime is currently inactive.");
- }
-
- /**
- * Print current stack frame variables
- * @param frameVariables Current frame variables
- */
- public void printFrameVariables(LocalVariableMap frameVariables)
- {
- //Display local stack frame variables
- if (frameVariables != null && !frameVariables.keySet().isEmpty()) {
- System.out.println(" Local variables:");
- System.out.format("\t%-40s %-40s", "Variable name", "Variable value" );
- for( String varname : frameVariables.keySet())
- System.out.format("\n\t%-40s %-40s", varname, frameVariables.get(varname).toString());
- System.out.println();
- }
- else
- System.out.println("\tSymbol table for current frame is empty");
- }
-
- /**
- * Print current stack frame information
- * @param frame Current stack frame which contains pc and local variables
- */
- public void printFrame(DMLFrame frame) {
- if (frame != null && frame.getPC() != null) {
- printPC(frame.getPC());
- printFrameVariables(frame.getVariables());
- }
- else
- System.out.println("DML runtime is currently inactive.");
- }
-
- /**
- * Print current call stack
- * @param currentEC Current stack frame
- * @param callStack Saved stack frames
- */
- public void printCallStack(DMLFrame currFrame, Stack<DMLFrame> callStack) {
- int frameID = 0;
- if (currFrame == null)
- System.out.println("DML runtime is currently inactive.");
- else {
- if (callStack != null) {
- for(DMLFrame frame : callStack) {
- System.out.println("Frame id: " + frameID++);
- printFrame(frame);
- }
- }
- System.out.println("Current frame id: " + frameID++);
- printFrame(currFrame);
- }
- }
-
- /**
- * Print DML scalar variable in current frame (if existing)
- * @param variables Current frame variables
- * @param varname Variable name
- */
- public void printScalarVariable(LocalVariableMap variables, String varname) {
- if (varname == null) {
- System.err.println("No scalar variable name entered.");
- return;
- }
- if (variables != null && !variables.keySet().isEmpty()) {
- if (variables.get(varname) != null) {
- if (variables.get(varname).getDataType() == DataType.SCALAR)
- System.out.println(varname + " = " + variables.get(varname).toString());
- else
- System.out.println("Variable \""+varname+"\" is not scalar variable.");
- }
- else
- System.out.println("DML scalar variable \""+varname+"\" is not in the current frame scope. Try \"a\" to list all variables within current frame scope.");
- }
- else
- System.out.println("Symbol table for current frame is empty");
- }
-
- /**
- * Set value of specified DML scalar variable in current frame (if existing)
- * @param variables Current frame variables
- * @param args CLI arguments for current debugger function (Variable name, and value)
- */
- public void setScalarValue(LocalVariableMap variables, String [] args) {
- String varname = args[0];
- if (variables != null && !variables.keySet().isEmpty()) {
- if (variables.get(varname) != null) {
- if (variables.get(varname).getDataType() == DataType.SCALAR) {
- Data value;
- //try {
- switch(variables.get(varname).getValueType()) {
- case DOUBLE:
- double d = Double.parseDouble(args[1]);
- value = (ScalarObject) new DoubleObject(d);
- break;
- case INT:
- long i = Long.parseLong(args[1]);
- value = (ScalarObject) new IntObject(i);
- break;
- case BOOLEAN:
- boolean b = Boolean.parseBoolean(args[1]);
- value = (ScalarObject) new BooleanObject(b);
- break;
- case STRING:
- value = (ScalarObject) new StringObject(args[1]);
- break;
- default:
- System.err.println("Invalid scalar value type.");
- return;
- }
- //}
-// catch (Exception e) {
-// System.err.println("Error processing set scalar value command for variable"+varname+".");
-// return;
-// }
- variables.put(varname, value);
- System.out.println(varname + " = " + variables.get(varname).toString());
- }
- else
- System.out.println("Variable \""+varname+"\" is not scalar variable.");
- }
- else
- System.out.println("DML scalar variable \""+varname+"\" is not in the current frame scope. Try \"a\" to list all variables within current frame scope.");
- }
- else
- System.out.println("Symbol table for current frame is empty");
- }
-
- public void print(LocalVariableMap variables, String varname, String displayFunction, int rowIndex, int colIndex) throws DMLRuntimeException {
- if (varname == null) {
- System.err.println("No matrix variable name entered.");
- return;
- }
-
- if (variables != null && !variables.keySet().isEmpty()) {
- if (variables.get(varname) != null) {
- if (variables.get(varname).getDataType() == DataType.MATRIX) {
- try {
- MatrixObject mo = null;
-
- try {
- mo = (MatrixObject) variables.get(varname);
- if (mo.getStatusAsString().equals("EMPTY") && (OptimizerUtils.estimateSizeExactSparsity(mo.getNumRows(), mo.getNumColumns(), mo.getSparsity()) > OptimizerUtils.getLocalMemBudget())) {
- //TODO @jlugoma Need to add functionality to bring and display a block.
- System.err.println("ERROR: Matrix dimensions are too large to fit in main memory.");
- return;
- }
- }
- catch(Exception fetchMatrixException) {
- System.err.println("ERROR: While fetching the matrix from symbol table.");
- return;
- }
-
- if(displayFunction.compareTo("value") == 0) {
- MatrixBlock mb = null;
- try {
- // Only read the MatrixBlock when asked to print, but not for whatis
- mb = mo.acquireRead();
- prettyPrintMatrixBlock(mb, rowIndex, colIndex);
- mo.release();
- }
- catch(Exception fetchMatrixException) {
- System.err.println("ERROR: Matrix dimensions are too large to fit in main memory.");
- return;
- }
- }
- else if(displayFunction.compareTo("metadata") == 0) {
- System.out.println("Metadata of " + varname + ": matrix"+variables.get(varname).getMetaData().toString());
- }
-
-
- } catch (Exception e) {
- String command = "";
- if(displayFunction.compareTo("value") == 0) {
- command = "print";
- }
- else {
- command = "whatis";
- }
- System.err.println("Error processing \'" + command + "\' command for variable "+varname+".");
- return;
- }
- }
- else if (variables.get(varname).getDataType() == DataType.SCALAR) {
- if(displayFunction.compareTo("value") == 0) {
- System.out.println(varname + " = " + variables.get(varname).toString());
- }
- else if(displayFunction.compareTo("metadata") == 0) {
- System.out.println("Metadata of " + varname + ": DataType.SCALAR");
- }
- }
- else
- System.out.println("Variable \""+varname+"\" is not a matrix or vector or scalar variable.");
- }
- else
- System.out.println("DML matrix variable \""+varname+"\" is not in the current frame scope. Try \"info frame\" to list all variables within current frame scope.");
- }
- else
- System.out.println("Symbol table for current frame is empty");
- }
-
-
-
- /**
- * Print DML matrix variable in current frame (if existing)
- * @param variables Current frame variables
- * @param varname Variable name
- * @throws DMLRuntimeException
- */
- public void printMatrixVariable(LocalVariableMap variables, String varname) throws DMLRuntimeException {
- if (varname == null) {
- System.err.println("No matrix variable name entered.");
- return;
- }
- if (variables != null && !variables.keySet().isEmpty()) {
- if (variables.get(varname) != null) {
- if (variables.get(varname).getDataType() == DataType.MATRIX) {
- try {
- MatrixObject mo = (MatrixObject) variables.get(varname);
- if (mo.getStatusAsString().equals("EMPTY") && (OptimizerUtils.estimateSizeExactSparsity(mo.getNumRows(), mo.getNumColumns(), mo.getSparsity()) > OptimizerUtils.getLocalMemBudget())) {
- //TODO @jlugoma Need to add functionality to bring and display a block.
- System.err.println("ERROR: DML matrix/vector dimensions are too large to fit in main memory.");
- return;
- }
- MatrixBlock mb = mo.acquireRead();
- prettyPrintMatrixBlock(mb, -1, -1);
- mo.release();
- if (mb.getNumRows() > DISPLAY_MAX_ROWS || mb.getNumColumns() > DISPLAY_MAX_COLUMNS) {
- System.out.format("WARNING: DML matrix/vector is too large to display on the screen."
- + "\nOnly a snapshot of %d row(s) and %d column(s) is being displayed.\n",
- min(mb.getNumRows(), DISPLAY_MAX_ROWS), min(mb.getNumColumns(), DISPLAY_MAX_COLUMNS));
- }
- System.out.println("Metadata: "+variables.get(varname).getMetaData().toString());
- } catch (Exception e) {
- System.err.println("Error processing display DML matrix command for variable "+varname+".");
- return;
- }
- }
- else
- System.out.println("Variable \""+varname+"\" is not a matrix or vector variable.");
- }
- else
- System.out.println("DML matrix variable \""+varname+"\" is not in the current frame scope. Try \"a\" to list all variables within current frame scope.");
- }
- else
- System.out.println("Symbol table for current frame is empty");
- }
-
- /**
- * Print DML matrix cell contents in current frame (if existing)
- * @param variables Current frame variables
- * @param args CLI arguments for current debugger function (Variable name, and row and column indexes)
- */
- public void printMatrixCell(LocalVariableMap variables, String [] args) {
- String varname = args[0];
- int rowIndex, columnIndex;
- try {
- rowIndex = Integer.parseInt(args[1]);
- columnIndex = Integer.parseInt(args[2]);
- } catch (Exception e) {
- System.err.print("Invalid display cell arguments.");
- return;
- }
- if (variables != null && !variables.keySet().isEmpty()) {
- if (variables.get(varname) != null) {
- if (variables.get(varname).getDataType() == DataType.MATRIX) {
- double cellValue;
- try {
- MatrixObject mo = (MatrixObject) variables.get(varname);
- if (mo.getStatusAsString().equals("EMPTY") && (OptimizerUtils.estimateSizeExactSparsity(mo.getNumRows(), mo.getNumColumns(), mo.getSparsity()) > OptimizerUtils.getLocalMemBudget())) {
- //TODO @jlugoma Need to add functionality to bring and display a block.
- System.err.println("ERROR: DML matrix/vector dimensions are too large to fit in main memory.");
- return;
- }
- MatrixBlock mb = mo.acquireRead();
- cellValue = mb.getValue(rowIndex, columnIndex);
- mo.release();
- } catch (Exception e) {
- System.err.println("Error processing DML matrix variable "+varname+". Certain matrix operations are disabled due to memory constraints or read-only restrictions.");
- return;
- }
- System.out.println(varname+"["+rowIndex+","+columnIndex+"] = "+cellValue);
- }
- else
- System.out.println("Variable \""+varname+"\" is not a matrix or vector variable.");
- }
- else
- System.out.println("DML matrix variable \""+varname+"\" is not in the current frame scope. Try \"a\" to list all variables within current frame scope.");
- }
- else
- System.out.println("Symbol table for current frame is empty");
- }
-
- /**
- * Set DML matrix cell contents in current frame (if existing)
- * @param variables Current frame variables
- * @param args CLI arguments for current debugger function (Variable name, and row and column indexes)
- */
- public void setMatrixCell(LocalVariableMap variables, String [] args) {
- String varname = args[0];
- int rowIndex, columnIndex;
- double value;
- try {
- rowIndex = Integer.parseInt(args[1]) - 1;
- columnIndex = Integer.parseInt(args[2]) - 1;
- value = Double.parseDouble(args[3]);
- } catch (Exception e) {
- System.err.print("Invalid set cell arguments.");
- return;
- }
- if (variables != null && !variables.keySet().isEmpty()) {
- if (variables.get(varname) != null) {
- if (variables.get(varname).getDataType() == DataType.MATRIX) {
- double updatedCellValue;
- try {
- MatrixObject mo = (MatrixObject) variables.get(varname);
- if (mo.getStatusAsString().equals("EMPTY") && (OptimizerUtils.estimateSizeExactSparsity(mo.getNumRows(), mo.getNumColumns(), mo.getSparsity()) > OptimizerUtils.getLocalMemBudget())) {
- //TODO @jlugoma Need to add functionality to bring and display a block.
- System.err.println("ERROR: DML matrix/vector dimensions are too large to fit in main memory.");
- return;
- }
- MatrixBlock mb = mo.acquireModify();
- mb.setValue(rowIndex, columnIndex, value);
- updatedCellValue = mb.getValue(rowIndex, columnIndex);
- mo.release();
- } catch (Exception e) {
- System.err.println("Error processing DML matrix variable "+varname+". Certain matrix operations are disabled due to memory constraints or read-only restrictions.");
- return;
- }
- System.out.println(varname+"["+ (rowIndex+1) +","+ (columnIndex+1) +"] = "+updatedCellValue);
- }
- else
- System.out.println("Variable \""+varname+"\" is not a matrix or vector variable.");
- }
- else
- System.out.println("DML matrix variable \""+varname+"\" is not in the current frame scope. Try \"a\" to list all variables within current frame scope.");
- }
- else
- System.out.println("Symbol table for current frame is empty");
- }
-
- /////////////////////////////////////////
- // internal debugger parsing functions //
- /////////////////////////////////////////
-
- /**
- * Parse command line argument and return valid value
- * @param args CLI arguments for current debugger function
- * @return Validated value for current debug command
- */
- protected String getValue(String[] args) {
- String value = null;
- if (args != null) {
- if (args.length > 1)
- System.err.println("Invalid number of argument values for this command. Try \"help\".");
- value = args[0];
- }
- return value;
- }
-
- /**
- * Parse command line argument and return valid integer
- * @param args CLI arguments for current debugger function
- * @param length Maximum value for input parameter
- * @return Validated integer value for current debug command
- */
- protected int getValue(String[] args, int length) {
- int lineNum = 0;
- if (args == null || args.length > 1) {
- System.err.print("Invalid argument value for this command. Parameter must be a positive integer <= "+length);
- }
- else {
- try {
- lineNum = Integer.parseInt(args[0]);
- if (lineNum <= 0 || lineNum > length)
- {
- System.err.println("Invalid argument value for this command. Parameter must be a positive integer <= "+length);
- lineNum = 0;
- }
- } catch (NumberFormatException e) {
- System.err.println("Invalid integer format. Parameter must be a positive integer <= "+length);
- lineNum = 0;
- }
- }
- return lineNum;
- }
-
- /**
- * Parse command line arguments and return valid IntRange variable
- * @param args CLI arguments for range of debugger display functionality
- * @param size Size (number of lines of code) of DML script
- * @return Validated range of lines within DML script to be displayed
- * @throws DMLDebuggerException Invalid range
- */
- protected IntRange getRange(String [] args, int length) throws DMLDebuggerException {
- IntRange range = new IntRange(1, length);
- if (args == null)
- return range;
- if (args.length == 2) {
- try {
- range = new IntRange(Integer.parseInt(args[0]), Integer.parseInt(args[1]));
- if (range.getMinimumInteger() <= 0 || range.getMaximumInteger() > length) {
- System.err.println("Invalid range values. Parameters <start end> must be two positive integers.");
- range = new IntRange(0, 0);
- }
- } catch (NumberFormatException e) {
- System.err.println("Invalid integer range format. Parameter must be a positive integer <= "+length);
- range = new IntRange(0, 0);
- }
- }
- else {
- System.err.println("Invalid range values. Parameters <start end> must be two positive integers.");
- range = new IntRange(0, 0);
- }
- return range;
- }
-
- /**
- * Returns minimum between two integers
- * @param a Integer value
- * @param b Integer value
- * @return Minimum between a and b.
- */
- private int min(int a, int b) {
- if (a < b)
- return a;
- return b;
- }
-
- /**
- * Displays a pretty-printed version of a DML matrix or vector variable.
- * @param mb Current matrix block
- * @param rowIndex if rowIndex == -1, then prints all rows
- * @param colIndex if colIndex == -1, then prints all columns
- */
- private void prettyPrintMatrixBlock(MatrixBlock mb, int rowIndex, int colIndex) {
- if(rowIndex <= 0 && colIndex <= 0) {
- // Print entire matrix
- for(int i=0; i<min(mb.getNumRows(), DISPLAY_MAX_ROWS); i++) {
- for(int j=0; j<min(mb.getNumColumns(), DISPLAY_MAX_COLUMNS); j++) {
- System.out.format("%.4f\t", mb.quickGetValue(i, j));
- }
- System.out.println();
- }
- if (mb.getNumRows() > DISPLAY_MAX_ROWS || mb.getNumColumns() > DISPLAY_MAX_COLUMNS) {
- System.out.format("WARNING: DML matrix/vector is too large to display on the screen."
- + "\nOnly a snapshot of %d row(s) and %d column(s) is being displayed.\n",
- min(mb.getNumRows(), DISPLAY_MAX_ROWS), min(mb.getNumColumns(), DISPLAY_MAX_COLUMNS));
- }
- }
- else if(rowIndex >= 0 && colIndex >= 0) {
- // Print a cell
- System.out.format("%.4f\n", mb.quickGetValue(rowIndex-1, colIndex-1));
- }
- else if(rowIndex >= 0) {
- // Print a row
- //for(int i=0; i<min(mb.getNumRows(), DISPLAY_MAX_ROWS); i++) {
- for(int j=0; j<min(mb.getNumColumns(), DISPLAY_MAX_COLUMNS); j++) {
- System.out.format("%.4f\t", mb.quickGetValue(rowIndex-1, j));
- }
- System.out.println();
- if (mb.getNumColumns() > DISPLAY_MAX_COLUMNS) {
- System.out.format("WARNING: the row of given DML matrix/vector is too large to display on the screen."
- + "\nOnly a snapshot of %d column(s) is being displayed.\n",
- min(mb.getNumColumns(), DISPLAY_MAX_COLUMNS));
- }
- //}
- }
- else if(colIndex >= 0) {
- // Print a column
- for(int i=0; i<min(mb.getNumRows(), DISPLAY_MAX_ROWS); i++) {
- System.out.format("%.4f\t", mb.quickGetValue(i, colIndex-1));
- System.out.println();
- }
- if (mb.getNumRows() > DISPLAY_MAX_ROWS) {
- System.out.format("WARNING: the column of given DML matrix/vector is too large to display on the screen."
- + "\nOnly a snapshot of %d row(s) is being displayed.\n",
- min(mb.getNumRows(), DISPLAY_MAX_ROWS));
- }
- }
- }
-
- /**
- * Prepare current instruction for printing
- * by removing internal delimiters.
- * @param inst Instruction to be displayed
- * @return Post-processed instruction in string format
- */
- private static String prepareInstruction(String inst) {
- String tmp = inst;
- tmp = tmp.replaceAll(Lop.OPERAND_DELIMITOR, " ");
- tmp = tmp.replaceAll(Lop.DATATYPE_PREFIX, ".");
- tmp = tmp.replaceAll(Lop.INSTRUCTION_DELIMITOR, ", ");
-
- return tmp;
- }
-}