You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2016/03/27 22:41:23 UTC
[2/4] incubator-systemml git commit: [SYSTEMML-533] Fix variable name
placeholder for instruction patching
[SYSTEMML-533] Fix variable name placeholder for instruction patching
Whenever instructions expect constant value arguments but we have
variables during compilation, we generate placeholders and rely on
instruction patching during runtime. So far we used ## which runs into
collisions if used as literals. This patch changes this placeholder to
the extended ASCII character \u00b6 to avoid collisions and cleans up
remaining code that directly referred to the placeholder instead of the
global compiler constant.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/e0356496
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/e0356496
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/e0356496
Branch: refs/heads/master
Commit: e0356496adc2070c2d532fd242c7dbf263ab73e4
Parents: 67a2954
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Sat Mar 26 01:07:44 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Sat Mar 26 21:57:30 2016 -0700
----------------------------------------------------------------------
.../apache/sysml/hops/cost/CostEstimator.java | 12 ++--
.../java/org/apache/sysml/lops/LeftIndex.java | 35 -----------
src/main/java/org/apache/sysml/lops/Lop.java | 14 +++--
.../java/org/apache/sysml/lops/Ternary.java | 8 ---
.../ExternalFunctionProgramBlock.java | 6 --
.../org/apache/sysml/test/utils/TestUtils.java | 64 --------------------
6 files changed, 14 insertions(+), 125 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/e0356496/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java b/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
index 00be756..00790a4 100644
--- a/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
+++ b/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
@@ -419,8 +419,8 @@ public abstract class CostEstimator
{
String[] parts = InstructionUtils.getInstructionParts(st.nextToken());
byte outIndex = Byte.parseByte(parts[2]);
- long rlen = parts[3].contains("##")?-1:UtilFunctions.parseToLong(parts[3]);
- long clen = parts[4].contains("##")?-1:UtilFunctions.parseToLong(parts[4]);
+ long rlen = parts[3].contains(Lop.VARIABLE_NAME_PLACEHOLDER)?-1:UtilFunctions.parseToLong(parts[3]);
+ long clen = parts[4].contains(Lop.VARIABLE_NAME_PLACEHOLDER)?-1:UtilFunctions.parseToLong(parts[4]);
long brlen = Long.parseLong(parts[5]);
long bclen = Long.parseLong(parts[6]);
long nnz = (long) (Double.parseDouble(parts[9]) * rlen * clen);
@@ -501,9 +501,9 @@ public abstract class CostEstimator
protected String replaceInstructionPatch( String inst )
{
String ret = inst;
- while( ret.contains("##") ) {
- int index1 = ret.indexOf("##");
- int index2 = ret.indexOf("##", index1+3);
+ while( ret.contains(Lop.VARIABLE_NAME_PLACEHOLDER) ) {
+ int index1 = ret.indexOf(Lop.VARIABLE_NAME_PLACEHOLDER);
+ int index2 = ret.indexOf(Lop.VARIABLE_NAME_PLACEHOLDER, index1+3);
String replace = ret.substring(index1,index2+2);
ret = ret.replaceAll(replace, "1");
}
@@ -622,7 +622,7 @@ public abstract class CostEstimator
attr = new String[]{String.valueOf(paramsMap.get("margin").equals("rows")?0:1)};
}
- vs[0] = stats.get( parts[1].substring(7).replaceAll("##", "") );
+ vs[0] = stats.get( parts[1].substring(7).replaceAll(Lop.VARIABLE_NAME_PLACEHOLDER, "") );
vs[1] = _unknownStats; //TODO
vs[2] = stats.get( parts[parts.length-1] );
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/e0356496/src/main/java/org/apache/sysml/lops/LeftIndex.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/LeftIndex.java b/src/main/java/org/apache/sysml/lops/LeftIndex.java
index 5168421..0adf769 100644
--- a/src/main/java/org/apache/sysml/lops/LeftIndex.java
+++ b/src/main/java/org/apache/sysml/lops/LeftIndex.java
@@ -137,41 +137,6 @@ public class LeftIndex extends Lop
return sb.toString();
}
-/* @Override
- public String getInstructions(int input_index1, int input_index2, int input_index3, int input_index4, int input_index5, int output_index)
- throws LopsException {
-
- * Example: B = A[row_l:row_u, col_l:col_u]
- * A - input matrix (input_index1)
- * row_l - lower bound in row dimension
- * row_u - upper bound in row dimension
- * col_l - lower bound in column dimension
- * col_u - upper bound in column dimension
- *
- * Since row_l,row_u,col_l,col_u are scalars, values for input_index(2,3,4,5)
- * will be equal to -1. They should be ignored and the scalar value labels must
- * be derived from input lops.
-
- String rowl = this.getInputs().get(1).getOutputParameters().getLabel();
- if (this.getInputs().get(1).getExecLocation() != ExecLocation.Data
- || !((Data) this.getInputs().get(1)).isLiteral())
- rowl = "##" + rowl + "##";
- String rowu = this.getInputs().get(2).getOutputParameters().getLabel();
- if (this.getInputs().get(2).getExecLocation() != ExecLocation.Data
- || !((Data) this.getInputs().get(2)).isLiteral())
- rowu = "##" + rowu + "##";
- String coll = this.getInputs().get(3).getOutputParameters().getLabel();
- if (this.getInputs().get(3).getExecLocation() != ExecLocation.Data
- || !((Data) this.getInputs().get(3)).isLiteral())
- coll = "##" + coll + "##";
- String colu = this.getInputs().get(4).getOutputParameters().getLabel();
- if (this.getInputs().get(4).getExecLocation() != ExecLocation.Data
- || !((Data) this.getInputs().get(4)).isLiteral())
- colu = "##" + colu + "##";
-
- return getInstructions(Integer.toString(input_index1), rowl, rowu, coll, colu, Integer.toString(output_index));
- }
-*/
@Override
public String toString() {
return "leftIndex";
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/e0356496/src/main/java/org/apache/sysml/lops/Lop.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/Lop.java b/src/main/java/org/apache/sysml/lops/Lop.java
index bed9d10..1c2bd5b 100644
--- a/src/main/java/org/apache/sysml/lops/Lop.java
+++ b/src/main/java/org/apache/sysml/lops/Lop.java
@@ -77,13 +77,15 @@ public abstract class Lop
public static final String FILE_SEPARATOR = "/";
public static final String PROCESS_PREFIX = "_p";
- public static final String INSTRUCTION_DELIMITOR = "\u2021"; // "\u002c"; //",";
- public static final String OPERAND_DELIMITOR = "\u00b0"; //\u2021"; //00ea"; //"::#::";
- public static final String VALUETYPE_PREFIX = "\u00b7" ; //":#:";
- public static final String DATATYPE_PREFIX = VALUETYPE_PREFIX; //":#:";
- public static final String LITERAL_PREFIX = VALUETYPE_PREFIX; //":#:";
+ //special delimiters w/ extended ASCII characters to avoid collisions
+ public static final String INSTRUCTION_DELIMITOR = "\u2021";
+ public static final String OPERAND_DELIMITOR = "\u00b0";
+ public static final String VALUETYPE_PREFIX = "\u00b7" ;
+ public static final String DATATYPE_PREFIX = VALUETYPE_PREFIX;
+ public static final String LITERAL_PREFIX = VALUETYPE_PREFIX;
+ public static final String VARIABLE_NAME_PLACEHOLDER = "\u00b6";
+
public static final String NAME_VALUE_SEPARATOR = "="; // e.g., used in parameterized builtins
- public static final String VARIABLE_NAME_PLACEHOLDER = "##"; //TODO: use in LOPs
public static final String MATRIX_VAR_NAME_PREFIX = "_mVar";
public static final String SCALAR_VAR_NAME_PREFIX = "_Var";
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/e0356496/src/main/java/org/apache/sysml/lops/Ternary.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/Ternary.java b/src/main/java/org/apache/sysml/lops/Ternary.java
index 43f19bf..e697cc5 100644
--- a/src/main/java/org/apache/sysml/lops/Ternary.java
+++ b/src/main/java/org/apache/sysml/lops/Ternary.java
@@ -221,10 +221,6 @@ public class Ternary extends Lop
throw new LopsException(this.printErrorLocation() + "In Tertiary Lop, Unexpected input while computing the instructions for op: " + operation + " \n");
}
- // parse the third input (scalar)
- // if it is a literal, copy val, else surround with the label with
- // ## symbols. these will be replaced at runtime.
-
int scalarIndex = 2; // index of the scalar input
sb.append( "ctabletransformscalarweight" );
@@ -248,10 +244,6 @@ public class Ternary extends Lop
throw new LopsException(this.printErrorLocation() + "In Tertiary Lop, Unexpected input while computing the instructions for op: " + operation + " \n");
}
- // parse the third input (scalar)
- // if it is a literal, copy val, else surround with the label with
- // ## symbols. these will be replaced at runtime.
-
int scalarIndex2 = 1; // index of the scalar input
int scalarIndex3 = 2; // index of the scalar input
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/e0356496/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
index b36bd59..af1671a 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
@@ -492,12 +492,6 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
// create a GMR job that transforms each of these matrices from block to cell
for (int i = 0; i < matrices.size(); i++) {
- //inputs[i] = "##" + matrices.get(i).getName() + "##";
- //inputInfo[i] = binBlockInputInfo;
- //outputInfo[i] = textCellOutputInfo;
- //numRows[i] = numCols[i] = numRowsPerBlock[i] = numColsPerBlock[i] = -1;
- //resultDimsUnknown[i] = 1;
-
inLabels.add(matrices.get(i).getName());
outLabels.add(matrices.get(i).getName()+"_extFnInput");
resultIndex[i] = (byte) i; //(matrices.size()+i);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/e0356496/src/test/java/org/apache/sysml/test/utils/TestUtils.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/utils/TestUtils.java b/src/test/java/org/apache/sysml/test/utils/TestUtils.java
index e6e7c12..74a5667 100644
--- a/src/test/java/org/apache/sysml/test/utils/TestUtils.java
+++ b/src/test/java/org/apache/sysml/test/utils/TestUtils.java
@@ -44,8 +44,6 @@ import java.util.Iterator;
import java.util.Locale;
import java.util.Random;
import java.util.StringTokenizer;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
@@ -88,9 +86,6 @@ public class TestUtils
private static ArrayList<String> _AssertInfos = new ArrayList<String>();
private static boolean _AssertOccured = false;
- /** String used to replace variables in scripts */
- private static String _RS = "\\$\\$";
-
/**
* <p>
* Compares to arrays for equality. The elements in the array can be in
@@ -1635,65 +1630,6 @@ public class TestUtils
/**
* <p>
- * Replaces variables in a DML or R script with the specified values. A
- * variable of format ##name## will be replaced where the name is used to
- * identify the variable in the hashmap containing the belonging value.
- * </p>
- *
- * @param strScriptDirectory
- * directory which contains the DML script
- * @param strScriptFile
- * filename of the DML script
- * @param variables
- * hashmap containing all the variables and their replacements
- * @deprecated Use ParameterBuilder.setVariablesInScript instead
- */
- public static void setVariablesInScript(String strScriptDirectory, String strScriptFile,
- HashMap<String, String> variables) {
- try {
- String strScript = strScriptDirectory + strScriptFile;
- String strTmpScript = strScript + "t";
-
- BufferedReader in = new BufferedReader(new InputStreamReader(new FileInputStream(strScript)));
- FileOutputStream out = new FileOutputStream(strTmpScript);
- PrintWriter pw = new PrintWriter(out);
- String content;
- Pattern unresolvedVars = Pattern.compile(_RS + ".*" + _RS);
- /**
- * sothat variables, which were not assigned, are replaced by an
- * empty string
- */
- while ((content = in.readLine()) != null) {
- for (String variable : variables.keySet()) {
- Pattern pattern = Pattern.compile(_RS + variable + _RS);
- Matcher matcher = pattern.matcher(content);
- while (matcher.find()) {
- content = content.replaceFirst(matcher.group().replace("$", "\\$"), variables.get(variable));
- }
- }
- Matcher matcher = unresolvedVars.matcher(content);
- content = matcher.replaceAll("");
- pw.println(content);
- }
- pw.close();
- out.close();
- in.close();
-
- /*
- * // remove checksum files if created Path crcFile = new
- * Path(dmlScriptDirectory + "." + dmlScriptFile + ".crc"); if
- * (fs.exists(crcFile)) fs.delete(crcFile, false); crcFile = new
- * Path(dmlScriptDirectory + "." + dmlScriptFile + "t.crc"); if
- * (fs.exists(crcFile)) fs.delete(crcFile, false);
- */
- } catch (IOException e) {
- e.printStackTrace();
- fail("unable to set variables in dml script: " + e.getMessage());
- }
- }
-
- /**
- * <p>
* Prints out a DML script.
* </p>
*