You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/09/21 18:55:30 UTC
[1/4] systemml git commit: [SYSTEMML-1922] Fix memory-efficiency
spark broadcasts, OOM AutoEncoder
Repository: systemml
Updated Branches:
refs/heads/master 50a895f86 -> 32620f294
[SYSTEMML-1922] Fix memory-efficiency spark broadcasts, OOM AutoEncoder
This patch fixes OOM issues on our staging/autoencoder-2layer.dml
script, which revealed a general issue of unaccounted broadcast memory
requirements. In detail, Spark keeps a deep copy of the blockified
broadcast as well as a reference to the passed object. So far we did not
account for the latter, which keeps this object in-memory even though
its evicted and put onto a soft reference.
We now explicitly clear blocks (except in local mode) after the
broadcast has been created (and thus blockified) because remote fetches
only reference the blockified chunks.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/a3407ae7
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/a3407ae7
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/a3407ae7
Branch: refs/heads/master
Commit: a3407ae7a9d8156225db28d057a9e5db2cbc2c90
Parents: 50a895f
Author: Matthias Boehm <mb...@gmail.com>
Authored: Wed Sep 20 15:50:41 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu Sep 21 11:44:27 2017 -0700
----------------------------------------------------------------------
.../context/SparkExecutionContext.java | 34 +++++++++++++-------
.../spark/data/PartitionedBlock.java | 4 +++
2 files changed, 26 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/a3407ae7/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
index e0352b0..966049c 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
@@ -264,6 +264,10 @@ public class SparkExecutionContext extends ExecutionContext
return conf;
}
+
+ public static boolean isLocalMaster() {
+ return getSparkContextStatic().isLocal();
+ }
/**
* Spark instructions should call this for all matrix inputs except broadcast
@@ -491,21 +495,19 @@ public class SparkExecutionContext extends ExecutionContext
return rdd;
}
-
- /**
- * TODO So far we only create broadcast variables but never destroy
- * them. This is a memory leak which might lead to executor out-of-memory.
- * However, in order to handle this, we need to keep track when broadcast
- * variables are no longer required.
- *
- * @param varname variable name
- * @return wrapper for broadcast variables
- * @throws DMLRuntimeException if DMLRuntimeException occurs
- */
+
@SuppressWarnings("unchecked")
public PartitionedBroadcast<MatrixBlock> getBroadcastForVariable( String varname )
throws DMLRuntimeException
{
+ //NOTE: The memory consumption of this method is the in-memory size of the
+ //matrix object plus the partitioned size in 1k-1k blocks. Since the call
+ //to broadcast happens after the matrix object has been released, the memory
+ //requirements of blockified chunks in Spark's block manager are covered under
+ //this maximum. Also note that we explicitly clear the in-memory blocks once
+ //the broadcasts are created (other than in local mode) in order to avoid
+ //unnecessary memory requirements during the lifetime of this broadcast handle.
+
long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
MatrixObject mo = getMatrixObject(varname);
@@ -547,12 +549,16 @@ public class SparkExecutionContext extends ExecutionContext
int numBlks = Math.min(numPerPart, pmb.getNumRowBlocks()*pmb.getNumColumnBlocks()-offset);
PartitionedBlock<MatrixBlock> tmp = pmb.createPartition(offset, numBlks, new MatrixBlock());
ret[i] = getSparkContext().broadcast(tmp);
+ if( !isLocalMaster() )
+ tmp.clearBlocks();
}
}
else { //single partition
ret[0] = getSparkContext().broadcast(pmb);
+ if( !isLocalMaster() )
+ pmb.clearBlocks();
}
-
+
bret = new PartitionedBroadcast<MatrixBlock>(ret);
BroadcastObject<MatrixBlock> bchandle = new BroadcastObject<MatrixBlock>(bret, varname,
OptimizerUtils.estimatePartitionedSizeExactSparsity(mo.getMatrixCharacteristics()));
@@ -613,10 +619,14 @@ public class SparkExecutionContext extends ExecutionContext
int numBlks = Math.min(numPerPart, pmb.getNumRowBlocks()*pmb.getNumColumnBlocks()-offset);
PartitionedBlock<FrameBlock> tmp = pmb.createPartition(offset, numBlks, new FrameBlock());
ret[i] = getSparkContext().broadcast(tmp);
+ if( !isLocalMaster() )
+ tmp.clearBlocks();
}
}
else { //single partition
ret[0] = getSparkContext().broadcast(pmb);
+ if( !isLocalMaster() )
+ pmb.clearBlocks();
}
bret = new PartitionedBroadcast<FrameBlock>(ret);
http://git-wip-us.apache.org/repos/asf/systemml/blob/a3407ae7/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBlock.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBlock.java
index ca9cde4..08257f6 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBlock.java
@@ -254,6 +254,10 @@ public class PartitionedBlock<T extends CacheBlock> implements Externalizable
}
}
}
+
+ public void clearBlocks() {
+ _partBlocks = null;
+ }
/**
* Redirects the default java serialization via externalizable to our default
[2/4] systemml git commit: [SYTEMML-1924] Fix flaky right indexing
tests (incorrect R output)
Posted by mb...@apache.org.
[SYTEMML-1924] Fix flaky right indexing tests (incorrect R output)
This patch fixes intermittent test failures of the right indexing test
suite. These tests generate random data X and random indexing ranges
X[a:b,c:d]. The failures occurred whenever a==b because because R
implicitly converts the output (row vector) to a column vector. We now
check for these cases and explicitly transpose the R result.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/0bebfd86
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/0bebfd86
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/0bebfd86
Branch: refs/heads/master
Commit: 0bebfd869b6c715dfd9937a0916040053079e482
Parents: a3407ae
Author: Matthias Boehm <mb...@gmail.com>
Authored: Wed Sep 20 21:14:48 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu Sep 21 11:44:28 2017 -0700
----------------------------------------------------------------------
.../indexing/RightIndexingMatrixTest.java | 130 +++++++++----------
.../indexing/RightIndexingMatrixTest.R | 2 +
2 files changed, 63 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/0bebfd86/src/test/java/org/apache/sysml/test/integration/functions/indexing/RightIndexingMatrixTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/indexing/RightIndexingMatrixTest.java b/src/test/java/org/apache/sysml/test/integration/functions/indexing/RightIndexingMatrixTest.java
index 82f4d10..625dd50 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/indexing/RightIndexingMatrixTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/indexing/RightIndexingMatrixTest.java
@@ -33,10 +33,8 @@ import org.apache.sysml.test.integration.TestConfiguration;
import org.apache.sysml.test.utils.TestUtils;
-
public class RightIndexingMatrixTest extends AutomatedTestBase
{
-
private final static String TEST_NAME = "RightIndexingMatrixTest";
private final static String TEST_DIR = "functions/indexing/";
private final static String TEST_CLASS_DIR = TEST_DIR + RightIndexingMatrixTest.class.getSimpleName() + "/";
@@ -50,123 +48,117 @@ public class RightIndexingMatrixTest extends AutomatedTestBase
private final static double sparsity1 = 0.5;
private final static double sparsity2 = 0.01;
-
@Override
public void setUp() {
addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME,
- new String[] {"B", "C", "D"}));
+ new String[] {"B", "C", "D"}));
}
@Test
- public void testRightIndexingDenseCP()
- {
+ public void testRightIndexingDenseCP() {
runRightIndexingTest(ExecType.CP, false);
}
@Test
- public void testRightIndexingDenseSP()
- {
+ public void testRightIndexingDenseSP() {
runRightIndexingTest(ExecType.SPARK, false);
}
@Test
- public void testRightIndexingDenseMR()
- {
+ public void testRightIndexingDenseMR() {
runRightIndexingTest(ExecType.MR, false);
}
@Test
- public void testRightIndexingSparseCP()
- {
+ public void testRightIndexingSparseCP() {
runRightIndexingTest(ExecType.CP, true);
}
@Test
- public void testRightIndexingSparseSP()
- {
+ public void testRightIndexingSparseSP() {
runRightIndexingTest(ExecType.SPARK, true);
}
@Test
- public void testRightIndexingSparseMR()
- {
+ public void testRightIndexingSparseMR() {
runRightIndexingTest(ExecType.MR, true);
}
- /**
- *
- * @param et
- * @param sparse
- */
- public void runRightIndexingTest( ExecType et, boolean sparse )
+ //various regression tests that led to test failures before
+
+ @Test
+ public void testRightIndexingDenseCPFixed1() {
+ runRightIndexingTest(ExecType.CP, false, 2083, 2083, 437, 842);
+ }
+
+ @Test
+ public void testRightIndexingDenseCPFixed2() {
+ runRightIndexingTest(ExecType.CP, false, 1632, 1632, 282, 345);
+ }
+
+ public void runRightIndexingTest( ExecType et, boolean sparse ) {
+ Random rand = new Random(System.currentTimeMillis());
+ long rl = (long)(rand.nextDouble()*((double)rows))+1;
+ long ru = (long)(rand.nextDouble()*((double)(rows-rl+1)))+rl;
+ long cl = (long)(rand.nextDouble()*((double)cols))+1;
+ long cu = (long)(rand.nextDouble()*((double)(cols-cl+1)))+cl;
+
+ runRightIndexingTest(et, sparse, rl, ru, cl, cu);
+ }
+
+ public void runRightIndexingTest( ExecType et, boolean sparse, long rl, long ru, long cl, long cu )
{
- RUNTIME_PLATFORM oldRTP = rtplatform;
-
+ RUNTIME_PLATFORM platformOld = rtplatform;
+ switch( et ){
+ case MR: rtplatform = RUNTIME_PLATFORM.HADOOP; break;
+ case SPARK: rtplatform = RUNTIME_PLATFORM.SPARK; break;
+ default: rtplatform = RUNTIME_PLATFORM.HYBRID; break;
+ }
+
boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
+ if( rtplatform == RUNTIME_PLATFORM.SPARK )
+ DMLScript.USE_LOCAL_SPARK_CONFIG = true;
try
{
- TestConfiguration config = getTestConfiguration(TEST_NAME);
- if(et == ExecType.SPARK) {
- rtplatform = RUNTIME_PLATFORM.SPARK;
- }
- else {
- rtplatform = (et==ExecType.MR)? RUNTIME_PLATFORM.HADOOP : RUNTIME_PLATFORM.SINGLE_NODE;
- }
- if( rtplatform == RUNTIME_PLATFORM.SPARK )
- DMLScript.USE_LOCAL_SPARK_CONFIG = true;
-
-
- double sparsity = sparse ? sparsity2 : sparsity1;
-
- config.addVariable("rows", rows);
- config.addVariable("cols", cols);
-
- long rowstart=216, rowend=429, colstart=967, colend=1009;
- Random rand=new Random(System.currentTimeMillis());
- rowstart=(long)(rand.nextDouble()*((double)rows))+1;
- rowend=(long)(rand.nextDouble()*((double)(rows-rowstart+1)))+rowstart;
- colstart=(long)(rand.nextDouble()*((double)cols))+1;
- colend=(long)(rand.nextDouble()*((double)(cols-colstart+1)))+colstart;
- config.addVariable("rowstart", rowstart);
- config.addVariable("rowend", rowend);
- config.addVariable("colstart", colstart);
- config.addVariable("colend", colend);
+ TestConfiguration config = getTestConfiguration(TEST_NAME);
+ double sparsity = sparse ? sparsity2 : sparsity1;
+ config.addVariable("rows", rows);
+ config.addVariable("cols", cols);
+ config.addVariable("rowstart", rl);
+ config.addVariable("rowend", ru);
+ config.addVariable("colstart", cl);
+ config.addVariable("colend", cu);
loadTestConfiguration(config);
-
- /* This is for running the junit test the new way, i.e., construct the arguments directly */
+
String RI_HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = RI_HOME + TEST_NAME + ".dml";
- programArgs = new String[]{"-args", input("A"),
+ programArgs = new String[]{"-explain","-args", input("A"),
Long.toString(rows), Long.toString(cols),
- Long.toString(rowstart), Long.toString(rowend),
- Long.toString(colstart), Long.toString(colend),
+ Long.toString(rl), Long.toString(ru),
+ Long.toString(cl), Long.toString(cu),
output("B"), output("C"), output("D") };
fullRScriptName = RI_HOME + TEST_NAME + ".R";
rCmd = "Rscript" + " " + fullRScriptName + " " +
- inputDir() + " " + rowstart + " " + rowend + " " + colstart + " " + colend + " " + expectedDir();
-
+ inputDir() + " " + rl + " " + ru + " " + cl + " " + cu + " " + expectedDir();
+
double[][] A = getRandomMatrix(rows, cols, min, max, sparsity, System.currentTimeMillis());
- writeInputMatrix("A", A, true);
-
- boolean exceptionExpected = false;
- int expectedNumberOfJobs = -1;
- runTest(true, exceptionExpected, null, expectedNumberOfJobs);
+ writeInputMatrix("A", A, true);
+ //run tests
+ runTest(true, false, null, -1);
runRScript(true);
- //disableOutAndExpectedDeletion();
-
- for(String file: config.getOutputFiles())
- {
+
+ //compare results
+ for(String file: config.getOutputFiles()) {
HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS(file);
HashMap<CellIndex, Double> rfile = readRMatrixFromFS(file);
TestUtils.compareMatrices(dmlfile, rfile, epsilon, file+"-DML", file+"-R");
}
}
- finally
- {
- rtplatform = oldRTP;
+ finally {
+ rtplatform = platformOld;
DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/0bebfd86/src/test/scripts/functions/indexing/RightIndexingMatrixTest.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/indexing/RightIndexingMatrixTest.R b/src/test/scripts/functions/indexing/RightIndexingMatrixTest.R
index 62009d7..ed5b341 100644
--- a/src/test/scripts/functions/indexing/RightIndexingMatrixTest.R
+++ b/src/test/scripts/functions/indexing/RightIndexingMatrixTest.R
@@ -28,6 +28,8 @@ A1=readMM(paste(args[1], "A.mtx", sep=""))
A = as.matrix(A1);
B=A[args[2]:args[3],args[4]:args[5]]
+if( args[2]==args[3] )
+ B = t(B) #correct implicit row to column vector conversion
C=A[1:args[3],args[4]:ncol(A)]
D=A[,args[4]:args[5]]
writeMM(as(B,"CsparseMatrix"), paste(args[6], "B", sep=""), format="text")
[4/4] systemml git commit: [SYSTEMML-1927] New frame transformcolmap
builtin function
Posted by mb...@apache.org.
[SYSTEMML-1927] New frame transformcolmap builtin function
Our decision tree script requires dummy coded inputs of all categorical
attributes as well as a mapping matrix of their column positions. This
patch introduces a new frame builtin function that leverages the
transformencode meta data in order to automatically compute the mapping
matrix. Since this is a meta data operation, it is only implemented in
CP (but it requires the meta data frame to fit into memory).
Furthermore, this also includes a respective test case and the
documentation update.
This changes enables a future simplification of decision tree and random
forest, where we could do any pre-processing automatically inside the
script instead of requiring the user to do it manually.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/32620f29
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/32620f29
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/32620f29
Branch: refs/heads/master
Commit: 32620f29416f163a0ec7b498ed5307012d8cbf5c
Parents: d6139d1
Author: Matthias Boehm <mb...@gmail.com>
Authored: Thu Sep 21 00:55:04 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu Sep 21 11:54:11 2017 -0700
----------------------------------------------------------------------
docs/dml-language-reference.md | 1 +
src/main/java/org/apache/sysml/hops/Hop.java | 5 +-
.../sysml/hops/ParameterizedBuiltinOp.java | 15 ++-
.../apache/sysml/lops/ParameterizedBuiltin.java | 6 +-
.../org/apache/sysml/parser/DMLTranslator.java | 27 ++--
.../org/apache/sysml/parser/Expression.java | 2 +-
.../ParameterizedBuiltinFunctionExpression.java | 24 +++-
.../context/ExecutionContext.java | 4 +-
.../instructions/CPInstructionParser.java | 5 +-
.../cp/ParameterizedBuiltinCPInstruction.java | 18 ++-
.../sysml/runtime/transform/encode/Encoder.java | 13 ++
.../transform/encode/EncoderComposite.java | 24 ++++
.../transform/encode/EncoderDummycode.java | 20 +++
.../TransformFrameEncodeColmapTest.java | 122 +++++++++++++++++++
.../transform/TransformFrameEncodeColmap.dml | 39 ++++++
.../input/homes3/homes.tfspec_colmap1.json | 1 +
.../input/homes3/homes.tfspec_colmap2.json | 1 +
.../input/homes3/homes.tfspec_dummy2.json | 3 +-
.../functions/transform/ZPackageSuite.java | 1 +
19 files changed, 300 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/32620f29/docs/dml-language-reference.md
----------------------------------------------------------------------
diff --git a/docs/dml-language-reference.md b/docs/dml-language-reference.md
index d8ca07f..c402acc 100644
--- a/docs/dml-language-reference.md
+++ b/docs/dml-language-reference.md
@@ -1642,6 +1642,7 @@ Function | Description | Parameters | Example
transformencode() | Transforms a frame into a matrix using specification. <br/> Builds and applies frame metadata. | Input:<br/> target = <frame> <br/> spec = <json specification> <br/> Outputs: <matrix>, <frame>|[transformencode](dml-language-reference.html#transformencode)
transformdecode() | Transforms a matrix into a frame using specification. <br/> Valid only for specific transformation types. | Input:<br/> target = <matrix> <br/> spec = <json specification> <br/> meta = <frame> <br/> Output: <frame> |[transformdecode](dml-language-reference.html#transformdecode)
transformapply() | Transforms a frame into a matrix using specification. <br/> Applies existing frame metadata. | Input:<br/> target = <frame> <br/> spec = <json specification> <br/> meta = <frame> <br/> Output: <matrix> | [transformapply](dml-language-reference.html#transformapply)
+transformcolmap() | Obtains the column mapping of a transformed frame using the given specification. The input frame is assumed to be the meta data frame returned from a transformencode call. <br/> The output has a row per encoded input attribute, indicating the source column position, as well as the start and end positions in the encode output. | Input:<br/> target = <frame> <br/> spec = <json specification> <br/> Output: <matrix> |[transformcolmap](dml-language-reference.html#transformdecode)
The following table summarizes the supported transformations for <code>transformencode(), transformdecode(), transformapply()</code>. Note only recoding, dummy coding and pass-through are reversible, i.e., subject to <code>transformdecode()</code>, whereas binning, missing value imputation, and omit are not.
http://git-wip-us.apache.org/repos/asf/systemml/blob/32620f29/src/main/java/org/apache/sysml/hops/Hop.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/Hop.java b/src/main/java/org/apache/sysml/hops/Hop.java
index f8ee068..7e172ac 100644
--- a/src/main/java/org/apache/sysml/hops/Hop.java
+++ b/src/main/java/org/apache/sysml/hops/Hop.java
@@ -1118,7 +1118,7 @@ public abstract class Hop implements ParseInfo
public enum ParamBuiltinOp {
INVALID, CDF, INVCDF, GROUPEDAGG, RMEMPTY, REPLACE, REXPAND,
- TRANSFORMAPPLY, TRANSFORMDECODE, TRANSFORMMETA,
+ TRANSFORMAPPLY, TRANSFORMDECODE, TRANSFORMCOLMAP, TRANSFORMMETA,
TOSTRING
};
@@ -1388,8 +1388,9 @@ public abstract class Hop implements ParseInfo
HopsParameterizedBuiltinLops.put(ParamBuiltinOp.RMEMPTY, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.RMEMPTY);
HopsParameterizedBuiltinLops.put(ParamBuiltinOp.REPLACE, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.REPLACE);
HopsParameterizedBuiltinLops.put(ParamBuiltinOp.REXPAND, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.REXPAND);
- HopsParameterizedBuiltinLops.put(ParamBuiltinOp.TRANSFORMAPPLY, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.TRANSFORMAPPLY);
+ HopsParameterizedBuiltinLops.put(ParamBuiltinOp.TRANSFORMAPPLY, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.TRANSFORMAPPLY);
HopsParameterizedBuiltinLops.put(ParamBuiltinOp.TRANSFORMDECODE, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.TRANSFORMDECODE);
+ HopsParameterizedBuiltinLops.put(ParamBuiltinOp.TRANSFORMCOLMAP, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.TRANSFORMCOLMAP);
HopsParameterizedBuiltinLops.put(ParamBuiltinOp.TRANSFORMMETA, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.TRANSFORMMETA);
HopsParameterizedBuiltinLops.put(ParamBuiltinOp.TOSTRING, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.TOSTRING);
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/32620f29/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java b/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
index 80e8764..2d3e256 100644
--- a/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
+++ b/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
@@ -194,9 +194,10 @@ public class ParameterizedBuiltinOp extends Hop implements MultiThreadedHop
case CDF:
case INVCDF:
case REPLACE:
- case TRANSFORMAPPLY:
- case TRANSFORMDECODE:
- case TRANSFORMMETA:
+ case TRANSFORMAPPLY:
+ case TRANSFORMDECODE:
+ case TRANSFORMCOLMAP:
+ case TRANSFORMMETA:
case TOSTRING: {
ExecType et = optFindExecType();
ParameterizedBuiltin pbilop = new ParameterizedBuiltin(inputlops,
@@ -1070,6 +1071,7 @@ public class ParameterizedBuiltinOp extends Hop implements MultiThreadedHop
//force CP for in-memory only transform builtins
if( (_op == ParamBuiltinOp.TRANSFORMAPPLY && REMOTE==ExecType.MR)
|| _op == ParamBuiltinOp.TRANSFORMDECODE && REMOTE==ExecType.MR
+ || _op == ParamBuiltinOp.TRANSFORMCOLMAP
|| _op == ParamBuiltinOp.TRANSFORMMETA
|| _op == ParamBuiltinOp.TOSTRING
|| _op == ParamBuiltinOp.CDF
@@ -1162,13 +1164,18 @@ public class ParameterizedBuiltinOp extends Hop implements MultiThreadedHop
//TODO parse json spec
break;
}
-
case TRANSFORMAPPLY: {
//rows remain unchanged only if no omitting
//cols remain unchanged of no dummy coding
//TODO parse json spec
break;
}
+ case TRANSFORMCOLMAP: {
+ Hop target = getTargetHop();
+ setDim1( target.getDim2() );
+ setDim2( 3 ); //fixed schema
+ break;
+ }
default:
//do nothing
break;
http://git-wip-us.apache.org/repos/asf/systemml/blob/32620f29/src/main/java/org/apache/sysml/lops/ParameterizedBuiltin.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/ParameterizedBuiltin.java b/src/main/java/org/apache/sysml/lops/ParameterizedBuiltin.java
index fb48d2f..81b8f1f 100644
--- a/src/main/java/org/apache/sysml/lops/ParameterizedBuiltin.java
+++ b/src/main/java/org/apache/sysml/lops/ParameterizedBuiltin.java
@@ -36,10 +36,9 @@ import org.apache.sysml.parser.Expression.ValueType;
*/
public class ParameterizedBuiltin extends Lop
{
-
public enum OperationTypes {
CDF, INVCDF, RMEMPTY, REPLACE, REXPAND,
- TRANSFORMAPPLY, TRANSFORMDECODE, TRANSFORMMETA,
+ TRANSFORMAPPLY, TRANSFORMDECODE, TRANSFORMCOLMAP, TRANSFORMMETA,
TOSTRING
};
@@ -209,7 +208,8 @@ public class ParameterizedBuiltin extends Lop
case TRANSFORMAPPLY:
case TRANSFORMDECODE:
- case TRANSFORMMETA: {
+ case TRANSFORMCOLMAP:
+ case TRANSFORMMETA: {
sb.append(_operation.toString().toLowerCase()); //opcode
sb.append(OPERAND_DELIMITOR);
sb.append(compileGenericParamMap(_inputParams));
http://git-wip-us.apache.org/repos/asf/systemml/blob/32620f29/src/main/java/org/apache/sysml/parser/DMLTranslator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/DMLTranslator.java b/src/main/java/org/apache/sysml/parser/DMLTranslator.java
index 123fe19..e9d4187 100644
--- a/src/main/java/org/apache/sysml/parser/DMLTranslator.java
+++ b/src/main/java/org/apache/sysml/parser/DMLTranslator.java
@@ -2234,24 +2234,27 @@ public class DMLTranslator
case TRANSFORMAPPLY:
currBuiltinOp = new ParameterizedBuiltinOp(
- target.getName(), target.getDataType(),
- target.getValueType(), ParamBuiltinOp.TRANSFORMAPPLY,
- paramHops);
- break;
+ target.getName(), target.getDataType(), target.getValueType(),
+ ParamBuiltinOp.TRANSFORMAPPLY, paramHops);
+ break;
case TRANSFORMDECODE:
currBuiltinOp = new ParameterizedBuiltinOp(
- target.getName(), target.getDataType(),
- target.getValueType(), ParamBuiltinOp.TRANSFORMDECODE,
- paramHops);
- break;
+ target.getName(), target.getDataType(), target.getValueType(),
+ ParamBuiltinOp.TRANSFORMDECODE, paramHops);
+ break;
+
+ case TRANSFORMCOLMAP:
+ currBuiltinOp = new ParameterizedBuiltinOp(
+ target.getName(), target.getDataType(), target.getValueType(),
+ ParamBuiltinOp.TRANSFORMCOLMAP, paramHops);
+ break;
case TRANSFORMMETA:
currBuiltinOp = new ParameterizedBuiltinOp(
- target.getName(), target.getDataType(),
- target.getValueType(), ParamBuiltinOp.TRANSFORMMETA,
- paramHops);
- break;
+ target.getName(), target.getDataType(), target.getValueType(),
+ ParamBuiltinOp.TRANSFORMMETA, paramHops);
+ break;
case TOSTRING:
currBuiltinOp = new ParameterizedBuiltinOp(
http://git-wip-us.apache.org/repos/asf/systemml/blob/32620f29/src/main/java/org/apache/sysml/parser/Expression.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/Expression.java b/src/main/java/org/apache/sysml/parser/Expression.java
index 053a5a3..084850a 100644
--- a/src/main/java/org/apache/sysml/parser/Expression.java
+++ b/src/main/java/org/apache/sysml/parser/Expression.java
@@ -145,7 +145,7 @@ public abstract class Expression implements ParseInfo
GROUPEDAGG, RMEMPTY, REPLACE, ORDER,
// Distribution Functions
CDF, INVCDF, PNORM, QNORM, PT, QT, PF, QF, PCHISQ, QCHISQ, PEXP, QEXP,
- TRANSFORMAPPLY, TRANSFORMDECODE, TRANSFORMENCODE, TRANSFORMMETA,
+ TRANSFORMAPPLY, TRANSFORMDECODE, TRANSFORMENCODE, TRANSFORMCOLMAP, TRANSFORMMETA,
TOSTRING, // The "toString" method for DML; named arguments accepted to format output
INVALID
};
http://git-wip-us.apache.org/repos/asf/systemml/blob/32620f29/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java b/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java
index 1e28aed..e2cd5a2 100644
--- a/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java
+++ b/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java
@@ -69,6 +69,7 @@ public class ParameterizedBuiltinFunctionExpression extends DataIdentifier
opcodeMap.put("transformapply", Expression.ParameterizedBuiltinFunctionOp.TRANSFORMAPPLY);
opcodeMap.put("transformdecode", Expression.ParameterizedBuiltinFunctionOp.TRANSFORMDECODE);
opcodeMap.put("transformencode", Expression.ParameterizedBuiltinFunctionOp.TRANSFORMENCODE);
+ opcodeMap.put("transformcolmap", Expression.ParameterizedBuiltinFunctionOp.TRANSFORMCOLMAP);
opcodeMap.put("transformmeta", Expression.ParameterizedBuiltinFunctionOp.TRANSFORMMETA);
// toString
@@ -234,7 +235,11 @@ public class ParameterizedBuiltinFunctionExpression extends DataIdentifier
case TRANSFORMDECODE:
validateTransformDecode(output, conditional);
- break;
+ break;
+
+ case TRANSFORMCOLMAP:
+ validateTransformColmap(output, conditional);
+ break;
case TRANSFORMMETA:
validateTransformMeta(output, conditional);
@@ -322,6 +327,23 @@ public class ParameterizedBuiltinFunctionExpression extends DataIdentifier
output.setDimensions(-1, -1);
}
+ private void validateTransformColmap(DataIdentifier output, boolean conditional)
+ throws LanguageException
+ {
+ //validate data / metadata (recode maps)
+ Expression exprTarget = getVarParam(Statement.GAGG_TARGET);
+ checkDataType("transformcolmap", TF_FN_PARAM_DATA, DataType.FRAME, conditional);
+
+ //validate specification
+ checkDataValueType("transformcolmap", TF_FN_PARAM_SPEC, DataType.SCALAR, ValueType.STRING, conditional);
+ validateTransformSpec(TF_FN_PARAM_SPEC, conditional);
+
+ //set output dimensions
+ output.setDataType(DataType.MATRIX);
+ output.setValueType(ValueType.DOUBLE);
+ output.setDimensions(exprTarget.getOutput().getDim2(), 3);
+ }
+
private void validateTransformMeta(DataIdentifier output, boolean conditional)
throws LanguageException
{
http://git-wip-us.apache.org/repos/asf/systemml/blob/32620f29/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
index 3b2436e..7df465c 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
@@ -484,8 +484,8 @@ public class ExecutionContext {
{
MatrixObject mo = getMatrixObject(varName);
mo.acquireModify(outputData, opcode);
- mo.release(opcode);
- setVariable(varName, mo);
+ mo.release(opcode);
+ setVariable(varName, mo);
}
public void setMatrixOutput(String varName, MatrixBlock outputData, UpdateType flag, String opcode)
http://git-wip-us.apache.org/repos/asf/systemml/blob/32620f29/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java b/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
index f6b880d..80738c4 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
@@ -191,11 +191,12 @@ public class CPInstructionParser extends InstructionParser
String2CPInstructionType.put( "rmempty" , CPINSTRUCTION_TYPE.ParameterizedBuiltin);
String2CPInstructionType.put( "replace" , CPINSTRUCTION_TYPE.ParameterizedBuiltin);
String2CPInstructionType.put( "rexpand" , CPINSTRUCTION_TYPE.ParameterizedBuiltin);
+ String2CPInstructionType.put( "toString" , CPINSTRUCTION_TYPE.ParameterizedBuiltin);
String2CPInstructionType.put( "transformapply",CPINSTRUCTION_TYPE.ParameterizedBuiltin);
String2CPInstructionType.put( "transformdecode",CPINSTRUCTION_TYPE.ParameterizedBuiltin);
- String2CPInstructionType.put( "transformencode",CPINSTRUCTION_TYPE.MultiReturnParameterizedBuiltin);
+ String2CPInstructionType.put( "transformcolmap",CPINSTRUCTION_TYPE.ParameterizedBuiltin);
String2CPInstructionType.put( "transformmeta",CPINSTRUCTION_TYPE.ParameterizedBuiltin);
- String2CPInstructionType.put( "toString" , CPINSTRUCTION_TYPE.ParameterizedBuiltin);
+ String2CPInstructionType.put( "transformencode",CPINSTRUCTION_TYPE.MultiReturnParameterizedBuiltin);
// Variable Instruction Opcodes
String2CPInstructionType.put( "assignvar" , CPINSTRUCTION_TYPE.Variable);
http://git-wip-us.apache.org/repos/asf/systemml/blob/32620f29/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
index b7928a6..141df62 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
@@ -140,6 +140,7 @@ public class ParameterizedBuiltinCPInstruction extends ComputationCPInstruction
}
else if ( opcode.equals("transformapply")
|| opcode.equals("transformdecode")
+ || opcode.equals("transformcolmap")
|| opcode.equals("transformmeta"))
{
return new ParameterizedBuiltinCPInstruction(null, paramsMap, out, opcode, str);
@@ -254,7 +255,7 @@ public class ParameterizedBuiltinCPInstruction extends ComputationCPInstruction
else if ( opcode.equalsIgnoreCase("transformapply")) {
//acquire locks
FrameBlock data = ec.getFrameInput(params.get("target"));
- FrameBlock meta = ec.getFrameInput(params.get("meta"));
+ FrameBlock meta = ec.getFrameInput(params.get("meta"));
String[] colNames = data.getColumnNames();
//compute transformapply
@@ -266,7 +267,7 @@ public class ParameterizedBuiltinCPInstruction extends ComputationCPInstruction
ec.releaseFrameInput(params.get("target"));
ec.releaseFrameInput(params.get("meta"));
}
- else if ( opcode.equalsIgnoreCase("transformdecode")) {
+ else if ( opcode.equalsIgnoreCase("transformdecode")) {
//acquire locks
MatrixBlock data = ec.getMatrixInput(params.get("target"), getExtendedOpcode());
FrameBlock meta = ec.getFrameInput(params.get("meta"));
@@ -282,6 +283,19 @@ public class ParameterizedBuiltinCPInstruction extends ComputationCPInstruction
ec.releaseMatrixInput(params.get("target"), getExtendedOpcode());
ec.releaseFrameInput(params.get("meta"));
}
+ else if ( opcode.equalsIgnoreCase("transformcolmap")) {
+ //acquire locks
+ FrameBlock meta = ec.getFrameInput(params.get("target"));
+ String[] colNames = meta.getColumnNames();
+
+ //compute transformapply
+ Encoder encoder = EncoderFactory.createEncoder(params.get("spec"), colNames, meta.getNumColumns(), null);
+ MatrixBlock mbout = encoder.getColMapping(meta, new MatrixBlock(meta.getNumColumns(), 3, false));
+
+ //release locks
+ ec.setMatrixOutput(output.getName(), mbout, getExtendedOpcode());
+ ec.releaseFrameInput(params.get("target"));
+ }
else if ( opcode.equalsIgnoreCase("transformmeta")) {
//get input spec and path
String spec = getParameterMap().get("spec");
http://git-wip-us.apache.org/repos/asf/systemml/blob/32620f29/src/main/java/org/apache/sysml/runtime/transform/encode/Encoder.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/encode/Encoder.java b/src/main/java/org/apache/sysml/runtime/transform/encode/Encoder.java
index 4117c67..2c0855d 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/encode/Encoder.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/encode/Encoder.java
@@ -136,4 +136,17 @@ public abstract class Encoder implements Serializable
* @param meta frame block
*/
public abstract void initMetaData(FrameBlock meta);
+
+ /**
+ * Obtain the column mapping of encoded frames based on the passed
+ * meta data frame.
+ *
+ * @param meta meta data frame block
+ * @param out output matrix
+ * @return matrix with column mapping (one row per attribute)
+ */
+ public MatrixBlock getColMapping(FrameBlock meta, MatrixBlock out) {
+ //default: do nothing
+ return out;
+ }
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/32620f29/src/main/java/org/apache/sysml/runtime/transform/encode/EncoderComposite.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/encode/EncoderComposite.java b/src/main/java/org/apache/sysml/runtime/transform/encode/EncoderComposite.java
index c04a011..d777c62 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/encode/EncoderComposite.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/encode/EncoderComposite.java
@@ -117,6 +117,30 @@ public class EncoderComposite extends Encoder
}
@Override
+ public MatrixBlock getColMapping(FrameBlock meta, MatrixBlock out) {
+ //determine if dummycode encoder exists
+ EncoderDummycode dummy = null;
+ for( Encoder encoder : _encoders )
+ if( encoder instanceof EncoderDummycode )
+ dummy = (EncoderDummycode) encoder;
+ //computed shifted start positions
+ if( dummy != null ) {
+ //delete to dummycode encoder
+ out = dummy.getColMapping(meta, out);
+ }
+ //use simple 1-1 mapping
+ else {
+ for(int i=0; i<out.getNumRows(); i++) {
+ out.quickSetValue(i, 0, i+1);
+ out.quickSetValue(i, 1, i+1);
+ out.quickSetValue(i, 2, i+1);
+ }
+ }
+
+ return out;
+ }
+
+ @Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("CompositeEncoder("+_encoders.size()+"):\n");
http://git-wip-us.apache.org/repos/asf/systemml/blob/32620f29/src/main/java/org/apache/sysml/runtime/transform/encode/EncoderDummycode.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/encode/EncoderDummycode.java b/src/main/java/org/apache/sysml/runtime/transform/encode/EncoderDummycode.java
index 9d7a5e9..1cc9356 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/encode/EncoderDummycode.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/encode/EncoderDummycode.java
@@ -100,4 +100,24 @@ public class EncoderDummycode extends Encoder
_dummycodedLength += _domainSizes[j]-1;
}
}
+
+ @Override
+ public MatrixBlock getColMapping(FrameBlock meta, MatrixBlock out) {
+ final int clen = out.getNumRows();
+ for(int colID=1, idx=0, ncolID=1; colID <= clen; colID++) {
+ int start = ncolID;
+ if( idx < _colList.length && colID==_colList[idx] ) {
+ ncolID += meta.getColumnMetadata(colID-1).getNumDistinct();
+ idx ++;
+ }
+ else {
+ ncolID ++;
+ }
+ out.quickSetValue(colID-1, 0, colID);
+ out.quickSetValue(colID-1, 1, start);
+ out.quickSetValue(colID-1, 2, ncolID-1);
+ }
+
+ return out;
+ }
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/32620f29/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeColmapTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeColmapTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeColmapTest.java
new file mode 100644
index 0000000..5d19644
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeColmapTest.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.transform;
+
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.runtime.io.FrameReader;
+import org.apache.sysml.runtime.io.FrameReaderFactory;
+import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.junit.Test;
+
+public class TransformFrameEncodeColmapTest extends AutomatedTestBase
+{
+ private final static String TEST_NAME1 = "TransformFrameEncodeColmap";
+ private final static String TEST_DIR = "functions/transform/";
+ private final static String TEST_CLASS_DIR = TEST_DIR + TransformFrameEncodeColmapTest.class.getSimpleName() + "/";
+
+ //dataset and transform tasks without missing values
+ private final static String DATASET1 = "homes3/homes.csv";
+ private final static String SPEC1 = "homes3/homes.tfspec_colmap1.json";
+ private final static String SPEC1b = "homes3/homes.tfspec_colmap2.json";
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ addTestConfiguration(TEST_NAME1,
+ new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "y" }) );
+ }
+
+ @Test
+ public void testHomesIDsSingleNode() {
+ runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", false);
+ }
+
+ @Test
+ public void testHomesColnamesSingleNode() {
+ runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", true);
+ }
+
+ @Test
+ public void testHomesIDsSpark() {
+ runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", false);
+ }
+
+ @Test
+ public void testHomesColnamesSpark() {
+ runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", true);
+ }
+
+ private void runTransformTest( RUNTIME_PLATFORM rt, String ofmt, boolean colnames )
+ {
+ //set runtime platform
+ RUNTIME_PLATFORM rtold = rtplatform;
+ rtplatform = rt;
+
+ boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
+ if( rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK)
+ DMLScript.USE_LOCAL_SPARK_CONFIG = true;
+
+ //set transform specification
+ String DATASET = DATASET1;
+ String SPEC = colnames?SPEC1b:SPEC1;
+
+ if( !ofmt.equals("csv") )
+ throw new RuntimeException("Unsupported test output format");
+
+ try
+ {
+ getAndLoadTestConfiguration(TEST_NAME1);
+
+ String HOME = SCRIPT_DIR + TEST_DIR;
+ fullDMLScriptName = HOME + TEST_NAME1 + ".dml";
+ programArgs = new String[]{"-explain","-nvargs",
+ "DATA=" + HOME + "input/" + DATASET,
+ "TFSPEC=" + HOME + "input/" + SPEC,
+ "TFDATA=" + output("tfout"),
+ "OFMT=" + ofmt, "OSEP=," };
+
+ runTest(true, false, null, -1);
+
+ //read input/output and compare
+ FrameReader reader1 = FrameReaderFactory.createFrameReader(InputInfo.CSVInputInfo,
+ new CSVFileFormatProperties(true, ",", false));
+ FrameBlock fb1 = reader1.readFrameFromHDFS(HOME + "input/" + DATASET, -1L, -1L);
+ FrameReader reader2 = FrameReaderFactory.createFrameReader(InputInfo.CSVInputInfo);
+ FrameBlock fb2 = reader2.readFrameFromHDFS(output("tfout"), -1L, -1L);
+ String[][] R1 = DataConverter.convertToStringFrame(fb1);
+ String[][] R2 = DataConverter.convertToStringFrame(fb2);
+ TestUtils.compareFrames(R1, R2, R1.length, R1[0].length);
+ }
+ catch(Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ finally {
+ rtplatform = rtold;
+ DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/systemml/blob/32620f29/src/test/scripts/functions/transform/TransformFrameEncodeColmap.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/transform/TransformFrameEncodeColmap.dml b/src/test/scripts/functions/transform/TransformFrameEncodeColmap.dml
new file mode 100644
index 0000000..98ea7f2
--- /dev/null
+++ b/src/test/scripts/functions/transform/TransformFrameEncodeColmap.dml
@@ -0,0 +1,39 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+F1 = read($DATA);
+jspec = read($TFSPEC, data_type="scalar", value_type="string");
+
+[X, M] = transformencode(target=F1, spec=jspec);
+map = transformcolmap(target=M, spec=jspec);
+
+F2 = as.frame(matrix(0, nrow(F1), 1));
+for(i in 1:nrow(map)) {
+ T = X[, as.scalar(map[i,2]):as.scalar(map[i,3])];
+ if( ncol(T)==1 )
+ Tf = as.frame(T);
+ else
+ Tf = transformdecode(target=T, meta=M[,i], spec="{ids:true, dummycode:[1]}");
+ F2 = cbind(F2, Tf);
+}
+F2 = F2[,2:ncol(F2)];
+
+write(F2, $TFDATA, format=$OFMT);
http://git-wip-us.apache.org/repos/asf/systemml/blob/32620f29/src/test/scripts/functions/transform/input/homes3/homes.tfspec_colmap1.json
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/transform/input/homes3/homes.tfspec_colmap1.json b/src/test/scripts/functions/transform/input/homes3/homes.tfspec_colmap1.json
new file mode 100644
index 0000000..19385fa
--- /dev/null
+++ b/src/test/scripts/functions/transform/input/homes3/homes.tfspec_colmap1.json
@@ -0,0 +1 @@
+{ ids:true, "dummycode": [ 2, 7, 1 ] }
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/systemml/blob/32620f29/src/test/scripts/functions/transform/input/homes3/homes.tfspec_colmap2.json
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/transform/input/homes3/homes.tfspec_colmap2.json b/src/test/scripts/functions/transform/input/homes3/homes.tfspec_colmap2.json
new file mode 100644
index 0000000..acacbef
--- /dev/null
+++ b/src/test/scripts/functions/transform/input/homes3/homes.tfspec_colmap2.json
@@ -0,0 +1 @@
+{ "dummycode": [ "district", "view", "zipcode" ] }
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/systemml/blob/32620f29/src/test/scripts/functions/transform/input/homes3/homes.tfspec_dummy2.json
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/transform/input/homes3/homes.tfspec_dummy2.json b/src/test/scripts/functions/transform/input/homes3/homes.tfspec_dummy2.json
index 917fab6..acacbef 100644
--- a/src/test/scripts/functions/transform/input/homes3/homes.tfspec_dummy2.json
+++ b/src/test/scripts/functions/transform/input/homes3/homes.tfspec_dummy2.json
@@ -1,2 +1 @@
-{
- "dummycode": [ "district", "view", "zipcode" ] }
\ No newline at end of file
+{ "dummycode": [ "district", "view", "zipcode" ] }
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/systemml/blob/32620f29/src/test_suites/java/org/apache/sysml/test/integration/functions/transform/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/transform/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/transform/ZPackageSuite.java
index 2571031..21519ce 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/transform/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/transform/ZPackageSuite.java
@@ -33,6 +33,7 @@ import org.junit.runners.Suite;
TransformEncodeDecodeTest.class,
TransformFrameEncodeApplySubsetTest.class,
TransformFrameEncodeApplyTest.class,
+ TransformFrameEncodeColmapTest.class,
TransformFrameEncodeDecodeTest.class,
TransformFrameEncodeDecodeTokenTest.class,
})
[3/4] systemml git commit: [SYSTEMML-1925] Fix consistency SystemML
configuration properties
Posted by mb...@apache.org.
[SYSTEMML-1925] Fix consistency SystemML configuration properties
This patch creates consistency in terms of a common prefix for all
configuration properties. Before we used the prefixes dml, systemml, or
none, which is now consolidated to sysml.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/d6139d14
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/d6139d14
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/d6139d14
Branch: refs/heads/master
Commit: d6139d1470d72b2c21a82ddb7e1898cf6582ed6c
Parents: 0bebfd8
Author: Matthias Boehm <mb...@gmail.com>
Authored: Wed Sep 20 22:17:50 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu Sep 21 11:54:09 2017 -0700
----------------------------------------------------------------------
conf/SystemML-config.xml.template | 54 ++++++++++----------
docs/beginners-guide-caffe2dml.md | 8 +--
docs/standalone-guide.md | 24 ++++-----
.../apache/sysml/api/mlcontext/MLContext.java | 2 +-
.../java/org/apache/sysml/conf/DMLConfig.java | 53 ++++++++++---------
.../controlprogram/ParForProgramBlock.java | 21 --------
src/main/python/systemml/mlcontext.py | 2 +-
src/main/python/systemml/mllearn/estimators.py | 2 +-
src/main/standalone/SystemML-config.xml | 24 ++++-----
src/test/config/SystemML-config.xml | 24 ++++-----
.../functions/codegen/APICodegenTest.java | 3 +-
.../functions/mlcontext/GNMFTest.java | 3 +-
.../SystemML-config-codegen-compress.xml | 46 ++---------------
.../codegen/SystemML-config-codegen.xml | 44 ++--------------
.../codegen/SystemML-config-codegen6.xml | 44 ++--------------
.../compress/SystemML-config-compress.xml | 39 +-------------
.../functions/dmlscript/SystemML-config.xml | 10 ++--
.../gdfo/SystemML-config-globalopt.xml | 38 +-------------
18 files changed, 120 insertions(+), 321 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6139d14/conf/SystemML-config.xml.template
----------------------------------------------------------------------
diff --git a/conf/SystemML-config.xml.template b/conf/SystemML-config.xml.template
index ef24e30..6d5bf73 100644
--- a/conf/SystemML-config.xml.template
+++ b/conf/SystemML-config.xml.template
@@ -19,77 +19,77 @@
<root>
<!-- local fs tmp working directory-->
- <localtmpdir>/tmp/systemml</localtmpdir>
+ <sysml.localtmpdir>/tmp/systemml</sysml.localtmpdir>
<!-- hdfs tmp working directory-->
- <scratch>scratch_space</scratch>
+ <sysml.scratch>scratch_space</sysml.scratch>
<!-- compiler optimization level, valid values: 0 | 1 | 2 | 3 | 4, default: 2 -->
- <optlevel>2</optlevel>
+ <sysml.optlevel>2</sysml.optlevel>
<!-- default number of reduce tasks per MR job, default: 2 x number of nodes -->
- <numreducers>10</numreducers>
+ <sysml.numreducers>10</sysml.numreducers>
<!-- override jvm reuse flag for specific MR jobs, valid values: true | false -->
- <jvmreuse>false</jvmreuse>
+ <sysml.jvmreuse>false</sysml.jvmreuse>
<!-- default block dim for binary block files -->
- <defaultblocksize>1000</defaultblocksize>
+ <sysml.defaultblocksize>1000</sysml.defaultblocksize>
<!-- run systemml control program as yarn appmaster, in case of MR1 always falls back to client, please disable for debug mode -->
- <dml.yarn.appmaster>false</dml.yarn.appmaster>
+ <sysml.yarn.appmaster>false</sysml.yarn.appmaster>
<!-- maximum jvm heap size of the dml yarn appmaster in MB, the requested memory is 1.5x this parameter -->
- <dml.yarn.appmaster.mem>2048</dml.yarn.appmaster.mem>
+ <sysml.yarn.appmaster.mem>2048</sysml.yarn.appmaster.mem>
<!-- maximum jvm heap size of the map/reduce tasks in MB, the requested memory is 1.5x this parameter, negative values ignored -->
- <dml.yarn.mapreduce.mem>2048</dml.yarn.mapreduce.mem>
+ <sysml.yarn.mapreduce.mem>2048</sysml.yarn.mapreduce.mem>
<!-- yarn application submission queue, relevant for default capacity scheduler -->
- <dml.yarn.app.queue>default</dml.yarn.app.queue>
+ <sysml.yarn.app.queue>default</sysml.yarn.app.queue>
<!-- enables multi-threaded operations in singlenode control program -->
- <cp.parallel.ops>true</cp.parallel.ops>
+ <sysml.cp.parallel.ops>true</sysml.cp.parallel.ops>
<!-- enables multi-threaded read/write in singlenode control program -->
- <cp.parallel.io>true</cp.parallel.io>
+ <sysml.cp.parallel.io>true</sysml.cp.parallel.io>
<!-- enables compressed linear algebra, experimental feature -->
- <compressed.linalg>auto</compressed.linalg>
+ <sysml.compressed.linalg>auto</sysml.compressed.linalg>
<!-- enables operator fusion via code generation, experimental feature -->
- <codegen.enabled>false</codegen.enabled>
+ <sysml.codegen.enabled>false</sysml.codegen.enabled>
<!-- set the codegen java compiler (auto, janino, javac) -->
- <codegen.compiler>auto</codegen.compiler>
+ <sysml.codegen.compiler>auto</sysml.codegen.compiler>
<!-- if codegen.enabled, enables source code caching of fused operators -->
- <codegen.plancache>false</codegen.plancache>
+ <sysml.codegen.plancache>true</sysml.codegen.plancache>
<!-- if codegen.enabled, compile literals as constants: 1..heuristic, 2..always -->
- <codegen.literals>1</codegen.literals>
+ <sysml.codegen.literals>1</sysml.codegen.literals>
<!-- enables native blas for matrix multiplication and convolution, experimental feature (options: auto, mkl, openblas, none) -->
- <native.blas>none</native.blas>
+ <sysml.native.blas>none</sysml.native.blas>
<!-- prints finegrained statistics information -->
- <systemml.stats.finegrained>false</systemml.stats.finegrained>
+ <sysml.stats.finegrained>false</sysml.stats.finegrained>
<!-- prints extra statistics information for GPU -->
- <systemml.stats.extraGPU>false</systemml.stats.extraGPU>
+ <sysml.stats.extraGPU>false</sysml.stats.extraGPU>
<!-- prints extra statistics information for Deep Neural Networks done in CP mode -->
- <systemml.stats.extraDNN>false</systemml.stats.extraDNN>
+ <sysml.stats.extraDNN>false</sysml.stats.extraDNN>
<!-- sets the GPUs to use per process, -1 for all GPUs, a specific GPU number (5), a range (eg: 0-2) or a comma separated list (eg: 0,2,4)-->
- <systemml.gpu.availableGPUs>-1</systemml.gpu.availableGPUs>
+ <sysml.gpu.availableGPUs>-1</sysml.gpu.availableGPUs>
<!-- whether to synchronize GPUs after every GPU instruction -->
- <systemml.gpu.sync.postProcess>true</systemml.gpu.sync.postProcess>
+ <sysml.gpu.sync.postProcess>true</sysml.gpu.sync.postProcess>
<!-- whether to perform eager CUDA free on rmvar instruction -->
- <systemml.gpu.eager.cudaFree>false</systemml.gpu.eager.cudaFree>
-
- <!-- maximum wrap length for instruction and miscellaneous timer column of statistics -->
- <systemml.stats.maxWrapLength>30</systemml.stats.maxWrapLength>
+ <sysml.gpu.eager.cudaFree>false</sysml.gpu.eager.cudaFree>
+
+ <!-- maximum wrap length for instruction and miscellaneous timer column of statistics -->
+ <sysml.stats.maxWrapLength>30</sysml.stats.maxWrapLength>
</root>
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6139d14/docs/beginners-guide-caffe2dml.md
----------------------------------------------------------------------
diff --git a/docs/beginners-guide-caffe2dml.md b/docs/beginners-guide-caffe2dml.md
index 7671c32..12b21d4 100644
--- a/docs/beginners-guide-caffe2dml.md
+++ b/docs/beginners-guide-caffe2dml.md
@@ -152,8 +152,8 @@ Iter:2000, validation loss:173.66147359346, validation accuracy:97.4897540983606
- Print the generated DML script along with classification report: `lenet.set(debug=True)`
- Print the heavy hitters instruction and the execution plan (advanced users): `lenet.setStatistics(True).setExplain(True)`
-- (Optional but recommended) Enable [native BLAS](http://apache.github.io/systemml/native-backend): `lenet.setConfigProperty("native.blas", "auto")`
-- Enable experimental feature such as codegen: `lenet.setConfigProperty("codegen.enabled", "true").setConfigProperty("codegen.plancache", "true")`
+- (Optional but recommended) Enable [native BLAS](http://apache.github.io/systemml/native-backend): `lenet.setConfigProperty("sysml.native.blas", "auto")`
+- Enable experimental feature such as codegen: `lenet.setConfigProperty("sysml.codegen.enabled", "true").setConfigProperty("sysml.codegen.plancache", "true")`
- Force GPU execution (please make sure the required jcuda dependency are included): lenet.setGPU(True).setForceGPU(True)
Unlike Caffe where default train and test algorithm is `minibatch`, you can specify the
@@ -242,13 +242,13 @@ If you are using OpenBLAS, please ensure that it was built with `USE_OPENMP` fla
For more detail see http://apache.github.io/systemml/native-backend
```python
-caffe2dmlObject.setConfigProperty("native.blas", "auto")
+caffe2dmlObject.setConfigProperty("sysml.native.blas", "auto")
```
- Turn on the experimental codegen feature. This should help reduce unnecessary allocation cost after every binary operation.
```python
-caffe2dmlObject.setConfigProperty("codegen.enabled", "true").setConfigProperty("codegen.plancache", "true")
+caffe2dmlObject.setConfigProperty("sysml.codegen.enabled", "true").setConfigProperty("sysml.codegen.plancache", "true")
```
- Tuned the [Garbage Collector](http://spark.apache.org/docs/latest/tuning.html#garbage-collection-tuning).
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6139d14/docs/standalone-guide.md
----------------------------------------------------------------------
diff --git a/docs/standalone-guide.md b/docs/standalone-guide.md
index a401c30..7116f25 100644
--- a/docs/standalone-guide.md
+++ b/docs/standalone-guide.md
@@ -324,18 +324,18 @@ The `l2-svm-predict.dml` algorithm is used on our test data sample to predict th
The console output should show the accuracy of the trained model in percent, i.e.:
15/09/01 01:32:51 INFO api.DMLScript: BEGIN DML run 09/01/2015 01:32:51
- 15/09/01 01:32:51 INFO conf.DMLConfig: Updating localtmpdir with value /tmp/systemml
- 15/09/01 01:32:51 INFO conf.DMLConfig: Updating scratch with value scratch_space
- 15/09/01 01:32:51 INFO conf.DMLConfig: Updating optlevel with value 2
- 15/09/01 01:32:51 INFO conf.DMLConfig: Updating numreducers with value 10
- 15/09/01 01:32:51 INFO conf.DMLConfig: Updating jvmreuse with value false
- 15/09/01 01:32:51 INFO conf.DMLConfig: Updating defaultblocksize with value 1000
- 15/09/01 01:32:51 INFO conf.DMLConfig: Updating dml.yarn.appmaster with value false
- 15/09/01 01:32:51 INFO conf.DMLConfig: Updating dml.yarn.appmaster.mem with value 2048
- 15/09/01 01:32:51 INFO conf.DMLConfig: Updating dml.yarn.mapreduce.mem with value 2048
- 15/09/01 01:32:51 INFO conf.DMLConfig: Updating dml.yarn.app.queue with value default
- 15/09/01 01:32:51 INFO conf.DMLConfig: Updating cp.parallel.ops with value true
- 15/09/01 01:32:51 INFO conf.DMLConfig: Updating cp.parallel.io with value true
+ 15/09/01 01:32:51 INFO conf.DMLConfig: Updating sysml.localtmpdir with value /tmp/systemml
+ 15/09/01 01:32:51 INFO conf.DMLConfig: Updating sysml.scratch with value scratch_space
+ 15/09/01 01:32:51 INFO conf.DMLConfig: Updating sysml.optlevel with value 2
+ 15/09/01 01:32:51 INFO conf.DMLConfig: Updating sysml.numreducers with value 10
+ 15/09/01 01:32:51 INFO conf.DMLConfig: Updating sysml.jvmreuse with value false
+ 15/09/01 01:32:51 INFO conf.DMLConfig: Updating sysml.defaultblocksize with value 1000
+ 15/09/01 01:32:51 INFO conf.DMLConfig: Updating sysml.yarn.appmaster with value false
+ 15/09/01 01:32:51 INFO conf.DMLConfig: Updating sysml.yarn.appmaster.mem with value 2048
+ 15/09/01 01:32:51 INFO conf.DMLConfig: Updating sysml.yarn.mapreduce.mem with value 2048
+ 15/09/01 01:32:51 INFO conf.DMLConfig: Updating sysml.yarn.app.queue with value default
+ 15/09/01 01:32:51 INFO conf.DMLConfig: Updating sysml.parallel.ops with value true
+ 15/09/01 01:32:51 INFO conf.DMLConfig: Updating sysml.parallel.io with value true
Accuracy (%): 74.14965986394557
15/09/01 01:32:52 INFO api.DMLScript: SystemML Statistics:
Total execution time: 0.130 sec.
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6139d14/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java b/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
index 35720a5..b6b7eb0 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
@@ -291,7 +291,7 @@ public class MLContext {
/**
* Set configuration property, such as
- * {@code setConfigProperty("localtmpdir", "/tmp/systemml")}.
+ * {@code setConfigProperty("sysml.localtmpdir", "/tmp/systemml")}.
*
* @param propertyName
* property name
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6139d14/src/main/java/org/apache/sysml/conf/DMLConfig.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/conf/DMLConfig.java b/src/main/java/org/apache/sysml/conf/DMLConfig.java
index 857071d..56f96e4 100644
--- a/src/main/java/org/apache/sysml/conf/DMLConfig.java
+++ b/src/main/java/org/apache/sysml/conf/DMLConfig.java
@@ -61,36 +61,35 @@ public class DMLConfig
// external names of configuration properties
// (single point of change for all internal refs)
- public static final String LOCAL_TMP_DIR = "localtmpdir";
- public static final String SCRATCH_SPACE = "scratch";
- public static final String OPTIMIZATION_LEVEL = "optlevel";
- public static final String NUM_REDUCERS = "numreducers";
- public static final String JVM_REUSE = "jvmreuse";
- public static final String DEFAULT_BLOCK_SIZE = "defaultblocksize";
- public static final String YARN_APPMASTER = "dml.yarn.appmaster";
- public static final String YARN_APPMASTERMEM = "dml.yarn.appmaster.mem";
- public static final String YARN_MAPREDUCEMEM = "dml.yarn.mapreduce.mem";
- public static final String YARN_APPQUEUE = "dml.yarn.app.queue";
- public static final String CP_PARALLEL_OPS = "cp.parallel.ops";
- public static final String CP_PARALLEL_IO = "cp.parallel.io";
- public static final String COMPRESSED_LINALG = "compressed.linalg"; //auto, true, false
- public static final String NATIVE_BLAS = "native.blas";
- public static final String CODEGEN = "codegen.enabled"; //boolean
- public static final String CODEGEN_COMPILER = "codegen.compiler"; //see SpoofCompiler.CompilerType
- public static final String CODEGEN_PLANCACHE = "codegen.plancache"; //boolean
- public static final String CODEGEN_LITERALS = "codegen.literals"; //1..heuristic, 2..always
-
- public static final String EXTRA_FINEGRAINED_STATS = "systemml.stats.finegrained"; //boolean
- public static final String STATS_MAX_WRAP_LEN = "systemml.stats.maxWrapLength"; //int
- public static final String EXTRA_GPU_STATS = "systemml.stats.extraGPU"; //boolean
- public static final String EXTRA_DNN_STATS = "systemml.stats.extraDNN"; //boolean
- public static final String AVAILABLE_GPUS = "systemml.gpu.availableGPUs"; // String to specify which GPUs to use (a range, all GPUs, comma separated list or a specific GPU)
- public static final String SYNCHRONIZE_GPU = "systemml.gpu.sync.postProcess"; // boolean: whether to synchronize GPUs after every instruction
- public static final String EAGER_CUDA_FREE = "systemml.gpu.eager.cudaFree"; // boolean: whether to perform eager CUDA free on rmvar
+ public static final String LOCAL_TMP_DIR = "sysml.localtmpdir";
+ public static final String SCRATCH_SPACE = "sysml.scratch";
+ public static final String OPTIMIZATION_LEVEL = "sysml.optlevel";
+ public static final String NUM_REDUCERS = "sysml.numreducers";
+ public static final String JVM_REUSE = "sysml.jvmreuse";
+ public static final String DEFAULT_BLOCK_SIZE = "sysml.defaultblocksize";
+ public static final String YARN_APPMASTER = "sysml.yarn.appmaster";
+ public static final String YARN_APPMASTERMEM = "sysml.yarn.appmaster.mem";
+ public static final String YARN_MAPREDUCEMEM = "sysml.yarn.mapreduce.mem";
+ public static final String YARN_APPQUEUE = "sysml.yarn.app.queue";
+ public static final String CP_PARALLEL_OPS = "sysml.cp.parallel.ops";
+ public static final String CP_PARALLEL_IO = "sysml.cp.parallel.io";
+ public static final String COMPRESSED_LINALG = "sysml.compressed.linalg"; //auto, true, false
+ public static final String NATIVE_BLAS = "sysml.native.blas";
+ public static final String CODEGEN = "sysml.codegen.enabled"; //boolean
+ public static final String CODEGEN_COMPILER = "sysml.codegen.compiler"; //see SpoofCompiler.CompilerType
+ public static final String CODEGEN_PLANCACHE = "sysml.codegen.plancache"; //boolean
+ public static final String CODEGEN_LITERALS = "sysml.codegen.literals"; //1..heuristic, 2..always
+ public static final String EXTRA_FINEGRAINED_STATS = "sysml.stats.finegrained"; //boolean
+ public static final String STATS_MAX_WRAP_LEN = "sysml.stats.maxWrapLength"; //int
+ public static final String EXTRA_GPU_STATS = "sysml.stats.extraGPU"; //boolean
+ public static final String EXTRA_DNN_STATS = "sysml.stats.extraDNN"; //boolean
+ public static final String AVAILABLE_GPUS = "sysml.gpu.availableGPUs"; // String to specify which GPUs to use (a range, all GPUs, comma separated list or a specific GPU)
+ public static final String SYNCHRONIZE_GPU = "sysml.gpu.sync.postProcess"; // boolean: whether to synchronize GPUs after every instruction
+ public static final String EAGER_CUDA_FREE = "sysml.gpu.eager.cudaFree"; // boolean: whether to perform eager CUDA free on rmvar
// Fraction of available memory to use. The available memory is computer when the GPUContext is created
// to handle the tradeoff on calling cudaMemGetInfo too often.
- public static final String GPU_MEMORY_UTILIZATION_FACTOR = "gpu.memory.util.factor";
+ public static final String GPU_MEMORY_UTILIZATION_FACTOR = "sysml.gpu.memory.util.factor";
// supported prefixes for custom map/reduce configurations
public static final String PREFIX_MAPRED = "mapred";
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6139d14/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
index 4dda1b4..07ca3ee 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
@@ -1330,27 +1330,6 @@ public class ParForProgramBlock extends ForProgramBlock
throws DMLRuntimeException
{
//TODO needs as precondition a systematic treatment of persistent read information.
- /*
- if( LIVEVAR_AWARE_CLEANUP && _sb != null)
- {
- //cleanup shared variables after they are unpinned
- VariableSet liveout = _sb.liveOut();
- for( Entry<String, Boolean> var : varState.entrySet() )
- {
- String varname = var.getKey();
- boolean unpinned = var.getValue();
- String fprefix = ConfigurationManager.getConfig().getTextValue("scratch")
- + Lop.FILE_SEPARATOR + Lop.PROCESS_PREFIX + DMLScript.getUUID();
-
- //delete unpinned vars if not in liveout (similar like rmvar) and not persistent input
- if( unpinned && !liveout.containsVariable(varname) )
-
- {
- VariableCPInstruction.processRemoveVariableInstruction(ec,varname);
- }
- }
- }
- */
}
/**
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6139d14/src/main/python/systemml/mlcontext.py
----------------------------------------------------------------------
diff --git a/src/main/python/systemml/mlcontext.py b/src/main/python/systemml/mlcontext.py
index 4a555f7..54e1969 100644
--- a/src/main/python/systemml/mlcontext.py
+++ b/src/main/python/systemml/mlcontext.py
@@ -792,7 +792,7 @@ class MLContext(object):
def setConfigProperty(self, propertyName, propertyValue):
"""
- Set configuration property, such as setConfigProperty("localtmpdir", "/tmp/systemml").
+ Set configuration property, such as setConfigProperty("sysml.localtmpdir", "/tmp/systemml").
Parameters
----------
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6139d14/src/main/python/systemml/mllearn/estimators.py
----------------------------------------------------------------------
diff --git a/src/main/python/systemml/mllearn/estimators.py b/src/main/python/systemml/mllearn/estimators.py
index 66f78be..1df1bb8 100644
--- a/src/main/python/systemml/mllearn/estimators.py
+++ b/src/main/python/systemml/mllearn/estimators.py
@@ -138,7 +138,7 @@ class BaseSystemMLEstimator(Estimator):
def setConfigProperty(self, propertyName, propertyValue):
"""
- Set configuration property, such as setConfigProperty("localtmpdir", "/tmp/systemml").
+ Set configuration property, such as setConfigProperty("sysml.localtmpdir", "/tmp/systemml").
Parameters
----------
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6139d14/src/main/standalone/SystemML-config.xml
----------------------------------------------------------------------
diff --git a/src/main/standalone/SystemML-config.xml b/src/main/standalone/SystemML-config.xml
index 9b52a6d..67e1d6c 100644
--- a/src/main/standalone/SystemML-config.xml
+++ b/src/main/standalone/SystemML-config.xml
@@ -19,38 +19,38 @@
<root>
<!-- local fs tmp working directory-->
- <localtmpdir>/tmp/systemml</localtmpdir>
+ <sysml.localtmpdir>/tmp/systemml</sysml.localtmpdir>
<!-- hdfs tmp working directory-->
- <scratch>scratch_space</scratch>
+ <sysml.scratch>scratch_space</sysml.scratch>
<!-- compiler optimization level, valid values: 0 | 1 | 2 | 3 | 4, default: 2 -->
- <optlevel>2</optlevel>
+ <sysml.optlevel>2</sysml.optlevel>
<!-- default number of reduce tasks per MR job, default: 2 x number of nodes -->
- <numreducers>10</numreducers>
+ <sysml.numreducers>10</sysml.numreducers>
<!-- override jvm reuse flag for specific MR jobs, valid values: true | false -->
- <jvmreuse>false</jvmreuse>
+ <sysml.jvmreuse>false</sysml.jvmreuse>
<!-- default block dim for binary block files -->
- <defaultblocksize>1000</defaultblocksize>
+ <sysml.defaultblocksize>1000</sysml.defaultblocksize>
<!-- run systemml control program as yarn appmaster, in case of MR1 always falls back to client, please disable for debug mode -->
- <dml.yarn.appmaster>false</dml.yarn.appmaster>
+ <sysml.yarn.appmaster>false</sysml.yarn.appmaster>
<!-- maximum jvm heap size of the dml yarn appmaster in MB, the requested memory is 1.5x this parameter -->
- <dml.yarn.appmaster.mem>2048</dml.yarn.appmaster.mem>
+ <sysml.yarn.appmaster.mem>2048</sysml.yarn.appmaster.mem>
<!-- maximum jvm heap size of the map/reduce tasks in MB, the requested memory is 1.5x this parameter, negative values ignored -->
- <dml.yarn.mapreduce.mem>2048</dml.yarn.mapreduce.mem>
+ <sysml.yarn.mapreduce.mem>2048</sysml.yarn.mapreduce.mem>
<!-- yarn application submission queue, relevant for default capacity scheduler -->
- <dml.yarn.app.queue>default</dml.yarn.app.queue>
+ <sysml.yarn.app.queue>default</sysml.yarn.app.queue>
<!-- enables multi-threaded matrix operations in singlenode control program -->
- <cp.parallel.ops>true</cp.parallel.ops>
+ <sysml.cp.parallel.ops>true</sysml.cp.parallel.ops>
<!-- enables multi-threaded read/write in singlenode control program -->
- <cp.parallel.io>true</cp.parallel.io>
+ <sysml.cp.parallel.io>true</sysml.cp.parallel.io>
</root>
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6139d14/src/test/config/SystemML-config.xml
----------------------------------------------------------------------
diff --git a/src/test/config/SystemML-config.xml b/src/test/config/SystemML-config.xml
index 3b25d99..2b708bd 100644
--- a/src/test/config/SystemML-config.xml
+++ b/src/test/config/SystemML-config.xml
@@ -19,38 +19,38 @@
<root>
<!-- local fs tmp working directory-->
- <localtmpdir>/tmp/systemml</localtmpdir>
+ <sysml.localtmpdir>/tmp/systemml</sysml.localtmpdir>
<!-- hdfs tmp working directory-->
- <scratch>scratch_space</scratch>
+ <sysml.scratch>scratch_space</sysml.scratch>
<!-- compiler optimization level, valid values: 0 | 1 | 2 | 3 | 4, default: 2 -->
- <optlevel>2</optlevel>
+ <sysml.optlevel>2</sysml.optlevel>
<!-- default number of reduce tasks per MR job, default: 2 x number of nodes -->
- <numreducers>10</numreducers>
+ <sysml.numreducers>10</sysml.numreducers>
<!-- override jvm reuse flag for specific MR jobs, valid values: true | false -->
- <jvmreuse>false</jvmreuse>
+ <sysml.jvmreuse>false</sysml.jvmreuse>
<!-- default block dim for binary block files -->
- <defaultblocksize>1000</defaultblocksize>
+ <sysml.defaultblocksize>1000</sysml.defaultblocksize>
<!-- run systemml control program as yarn appmaster, in case of MR1 always falls back to client, please disable for debug mode -->
- <dml.yarn.appmaster>false</dml.yarn.appmaster>
+ <sysml.yarn.appmaster>false</sysml.yarn.appmaster>
<!-- maximum jvm heap size of the dml yarn appmaster in MB, the requested memory is 1.5x this parameter -->
- <dml.yarn.appmaster.mem>2048</dml.yarn.appmaster.mem>
+ <sysml.yarn.appmaster.mem>2048</sysml.yarn.appmaster.mem>
<!-- maximum jvm heap size of the map/reduce tasks in MB, the requested memory is 1.5x this parameter, negative values ignored -->
- <dml.yarn.mapreduce.mem>2048</dml.yarn.mapreduce.mem>
+ <sysml.yarn.mapreduce.mem>2048</sysml.yarn.mapreduce.mem>
<!-- yarn application submission queue, relevant for default capacity scheduler -->
- <dml.yarn.app.queue>default</dml.yarn.app.queue>
+ <sysml.yarn.app.queue>default</sysml.yarn.app.queue>
<!-- enables multi-threaded matrix operations in singlenode control program -->
- <cp.parallel.ops>true</cp.parallel.ops>
+ <sysml.cp.parallel.ops>true</sysml.cp.parallel.ops>
<!-- enables multi-threaded read/write in singlenode control program -->
- <cp.parallel.io>true</cp.parallel.io>
+ <sysml.cp.parallel.io>true</sysml.cp.parallel.io>
</root>
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6139d14/src/test/java/org/apache/sysml/test/integration/functions/codegen/APICodegenTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/codegen/APICodegenTest.java b/src/test/java/org/apache/sysml/test/integration/functions/codegen/APICodegenTest.java
index 7e6ead1..de0a26f 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/codegen/APICodegenTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/codegen/APICodegenTest.java
@@ -29,6 +29,7 @@ import org.apache.sysml.api.jmlc.PreparedScript;
import org.apache.sysml.api.mlcontext.MLContext;
import org.apache.sysml.api.mlcontext.Script;
import org.apache.sysml.conf.CompilerConfig.ConfigType;
+import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.util.DataConverter;
@@ -91,7 +92,7 @@ public class APICodegenTest extends AutomatedTestBase
.setAppName("MLContextTest").setMaster("local");
JavaSparkContext sc = new JavaSparkContext(conf);
MLContext ml = new MLContext(sc);
- ml.setConfigProperty("codegen.enabled", "true");
+ ml.setConfigProperty(DMLConfig.CODEGEN, "true");
ml.setStatistics(true);
Script script = dml(s).in("X", mX).out("R");
ml.execute(script);
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6139d14/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/GNMFTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/GNMFTest.java b/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/GNMFTest.java
index 44f1f15..2f9489b 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/GNMFTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/GNMFTest.java
@@ -42,6 +42,7 @@ import org.apache.sysml.api.mlcontext.MatrixMetadata;
import org.apache.sysml.api.mlcontext.Script;
import org.apache.sysml.api.mlcontext.ScriptFactory;
import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.parser.ParseException;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.instructions.spark.utils.RDDConverterUtils;
@@ -182,7 +183,7 @@ public class GNMFTest extends MLContextTestBase
if(numRegisteredOutputs >= 2) {
script.out("W");
- ml.setConfigProperty("cp.parallel.ops", "false");
+ ml.setConfigProperty(DMLConfig.CP_PARALLEL_OPS, "false");
}
MLResults results = ml.execute(script);
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6139d14/src/test/scripts/functions/codegen/SystemML-config-codegen-compress.xml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/codegen/SystemML-config-codegen-compress.xml b/src/test/scripts/functions/codegen/SystemML-config-codegen-compress.xml
index 5c8a9b7..d8597fd 100644
--- a/src/test/scripts/functions/codegen/SystemML-config-codegen-compress.xml
+++ b/src/test/scripts/functions/codegen/SystemML-config-codegen-compress.xml
@@ -18,45 +18,9 @@
-->
<root>
- <!-- local fs tmp working directory-->
- <localtmpdir>/tmp/systemml</localtmpdir>
-
- <!-- hdfs tmp working directory-->
- <scratch>scratch_space</scratch>
-
- <!-- compiler optimization level, valid values: 0 | 1 | 2 | 3 | 4, default: 2 -->
- <optlevel>7</optlevel>
-
- <!-- default number of reduce tasks per MR job, default: 2 x number of nodes -->
- <numreducers>10</numreducers>
-
- <!-- override jvm reuse flag for specific MR jobs, valid values: true | false -->
- <jvmreuse>false</jvmreuse>
-
- <!-- default block dim for binary block files -->
- <defaultblocksize>1000</defaultblocksize>
-
- <!-- run systemml control program as yarn appmaster, in case of MR1 always falls back to client, please disable for debug mode -->
- <dml.yarn.appmaster>false</dml.yarn.appmaster>
-
- <!-- maximum jvm heap size of the dml yarn appmaster in MB, the requested memory is 1.5x this parameter -->
- <dml.yarn.appmaster.mem>2048</dml.yarn.appmaster.mem>
-
- <!-- maximum jvm heap size of the map/reduce tasks in MB, the requested memory is 1.5x this parameter, negative values ignored -->
- <dml.yarn.mapreduce.mem>2048</dml.yarn.mapreduce.mem>
-
- <!-- yarn application submission queue, relevant for default capacity scheduler -->
- <dml.yarn.app.queue>default</dml.yarn.app.queue>
-
- <!-- enables multi-threaded operations in singlenode control program -->
- <cp.parallel.ops>true</cp.parallel.ops>
-
- <!-- enables multi-threaded read/write in singlenode control program -->
- <cp.parallel.io>true</cp.parallel.io>
-
- <!-- enables automatic code generation -->
- <compressed.linalg>true</compressed.linalg>
- <codegen.enabled>true</codegen.enabled>
- <codegen.plancache>true</codegen.plancache>
- <codegen.literals>1</codegen.literals>
+ <sysml.optlevel>7</sysml.optlevel>
+ <sysml.compressed.linalg>true</sysml.compressed.linalg>
+ <sysml.codegen.enabled>true</sysml.codegen.enabled>
+ <sysml.codegen.plancache>true</sysml.codegen.plancache>
+ <sysml.codegen.literals>1</sysml.codegen.literals>
</root>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6139d14/src/test/scripts/functions/codegen/SystemML-config-codegen.xml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/codegen/SystemML-config-codegen.xml b/src/test/scripts/functions/codegen/SystemML-config-codegen.xml
index 95e0dba..3a00e70 100644
--- a/src/test/scripts/functions/codegen/SystemML-config-codegen.xml
+++ b/src/test/scripts/functions/codegen/SystemML-config-codegen.xml
@@ -18,44 +18,8 @@
-->
<root>
- <!-- local fs tmp working directory-->
- <localtmpdir>/tmp/systemml</localtmpdir>
-
- <!-- hdfs tmp working directory-->
- <scratch>scratch_space</scratch>
-
- <!-- compiler optimization level, valid values: 0 | 1 | 2 | 3 | 4, default: 2 -->
- <optlevel>7</optlevel>
-
- <!-- default number of reduce tasks per MR job, default: 2 x number of nodes -->
- <numreducers>10</numreducers>
-
- <!-- override jvm reuse flag for specific MR jobs, valid values: true | false -->
- <jvmreuse>false</jvmreuse>
-
- <!-- default block dim for binary block files -->
- <defaultblocksize>1000</defaultblocksize>
-
- <!-- run systemml control program as yarn appmaster, in case of MR1 always falls back to client, please disable for debug mode -->
- <dml.yarn.appmaster>false</dml.yarn.appmaster>
-
- <!-- maximum jvm heap size of the dml yarn appmaster in MB, the requested memory is 1.5x this parameter -->
- <dml.yarn.appmaster.mem>2048</dml.yarn.appmaster.mem>
-
- <!-- maximum jvm heap size of the map/reduce tasks in MB, the requested memory is 1.5x this parameter, negative values ignored -->
- <dml.yarn.mapreduce.mem>2048</dml.yarn.mapreduce.mem>
-
- <!-- yarn application submission queue, relevant for default capacity scheduler -->
- <dml.yarn.app.queue>default</dml.yarn.app.queue>
-
- <!-- enables multi-threaded operations in singlenode control program -->
- <cp.parallel.ops>true</cp.parallel.ops>
-
- <!-- enables multi-threaded read/write in singlenode control program -->
- <cp.parallel.io>true</cp.parallel.io>
-
- <!-- enables automatic code generation -->
- <codegen.enabled>true</codegen.enabled>
- <codegen.plancache>true</codegen.plancache>
- <codegen.literals>1</codegen.literals>
+ <sysml.optlevel>7</sysml.optlevel>
+ <sysml.codegen.enabled>true</sysml.codegen.enabled>
+ <sysml.codegen.plancache>true</sysml.codegen.plancache>
+ <sysml.codegen.literals>1</sysml.codegen.literals>
</root>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6139d14/src/test/scripts/functions/codegen/SystemML-config-codegen6.xml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/codegen/SystemML-config-codegen6.xml b/src/test/scripts/functions/codegen/SystemML-config-codegen6.xml
index fc41c2a..35d0956 100644
--- a/src/test/scripts/functions/codegen/SystemML-config-codegen6.xml
+++ b/src/test/scripts/functions/codegen/SystemML-config-codegen6.xml
@@ -18,44 +18,8 @@
-->
<root>
- <!-- local fs tmp working directory-->
- <localtmpdir>/tmp/systemml</localtmpdir>
-
- <!-- hdfs tmp working directory-->
- <scratch>scratch_space</scratch>
-
- <!-- compiler optimization level, valid values: 0 | 1 | 2 | 3 | 4, default: 2 -->
- <optlevel>6</optlevel>
-
- <!-- default number of reduce tasks per MR job, default: 2 x number of nodes -->
- <numreducers>10</numreducers>
-
- <!-- override jvm reuse flag for specific MR jobs, valid values: true | false -->
- <jvmreuse>false</jvmreuse>
-
- <!-- default block dim for binary block files -->
- <defaultblocksize>1000</defaultblocksize>
-
- <!-- run systemml control program as yarn appmaster, in case of MR1 always falls back to client, please disable for debug mode -->
- <dml.yarn.appmaster>false</dml.yarn.appmaster>
-
- <!-- maximum jvm heap size of the dml yarn appmaster in MB, the requested memory is 1.5x this parameter -->
- <dml.yarn.appmaster.mem>2048</dml.yarn.appmaster.mem>
-
- <!-- maximum jvm heap size of the map/reduce tasks in MB, the requested memory is 1.5x this parameter, negative values ignored -->
- <dml.yarn.mapreduce.mem>2048</dml.yarn.mapreduce.mem>
-
- <!-- yarn application submission queue, relevant for default capacity scheduler -->
- <dml.yarn.app.queue>default</dml.yarn.app.queue>
-
- <!-- enables multi-threaded operations in singlenode control program -->
- <cp.parallel.ops>true</cp.parallel.ops>
-
- <!-- enables multi-threaded read/write in singlenode control program -->
- <cp.parallel.io>true</cp.parallel.io>
-
- <!-- enables automatic code generation -->
- <codegen.enabled>true</codegen.enabled>
- <codegen.plancache>true</codegen.plancache>
- <codegen.literals>1</codegen.literals>
+ <sysml.optlevel>6</sysml.optlevel>
+ <sysml.codegen.enabled>true</sysml.codegen.enabled>
+ <sysml.codegen.plancache>true</sysml.codegen.plancache>
+ <sysml.codegen.literals>1</sysml.codegen.literals>
</root>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6139d14/src/test/scripts/functions/compress/SystemML-config-compress.xml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/compress/SystemML-config-compress.xml b/src/test/scripts/functions/compress/SystemML-config-compress.xml
index 4d56c66..9c4f4ba 100644
--- a/src/test/scripts/functions/compress/SystemML-config-compress.xml
+++ b/src/test/scripts/functions/compress/SystemML-config-compress.xml
@@ -18,42 +18,5 @@
-->
<root>
- <!-- local fs tmp working directory-->
- <localtmpdir>/tmp/systemml</localtmpdir>
-
- <!-- hdfs tmp working directory-->
- <scratch>scratch_space</scratch>
-
- <!-- compiler optimization level, valid values: 0 | 1 | 2 | 3 | 4, default: 2 -->
- <optlevel>2</optlevel>
-
- <!-- default number of reduce tasks per MR job, default: 2 x number of nodes -->
- <numreducers>10</numreducers>
-
- <!-- override jvm reuse flag for specific MR jobs, valid values: true | false -->
- <jvmreuse>false</jvmreuse>
-
- <!-- default block dim for binary block files -->
- <defaultblocksize>1000</defaultblocksize>
-
- <!-- run systemml control program as yarn appmaster, in case of MR1 always falls back to client, please disable for debug mode -->
- <dml.yarn.appmaster>false</dml.yarn.appmaster>
-
- <!-- maximum jvm heap size of the dml yarn appmaster in MB, the requested memory is 1.5x this parameter -->
- <dml.yarn.appmaster.mem>2048</dml.yarn.appmaster.mem>
-
- <!-- maximum jvm heap size of the map/reduce tasks in MB, the requested memory is 1.5x this parameter, negative values ignored -->
- <dml.yarn.mapreduce.mem>2048</dml.yarn.mapreduce.mem>
-
- <!-- yarn application submission queue, relevant for default capacity scheduler -->
- <dml.yarn.app.queue>default</dml.yarn.app.queue>
-
- <!-- enables multi-threaded matrix operations in singlenode control program -->
- <cp.parallel.ops>true</cp.parallel.ops>
-
- <!-- enables multi-threaded read/write in singlenode control program -->
- <cp.parallel.io>true</cp.parallel.io>
-
- <!-- enables compressed linear algebra for cp/spark -->
- <compressed.linalg>true</compressed.linalg>
+ <sysml.compressed.linalg>true</sysml.compressed.linalg>
</root>
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6139d14/src/test/scripts/functions/dmlscript/SystemML-config.xml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/dmlscript/SystemML-config.xml b/src/test/scripts/functions/dmlscript/SystemML-config.xml
index dc1e298..f0e4512 100644
--- a/src/test/scripts/functions/dmlscript/SystemML-config.xml
+++ b/src/test/scripts/functions/dmlscript/SystemML-config.xml
@@ -17,9 +17,9 @@
* under the License.
-->
<root>
-<numreducers>10</numreducers>
-<scratch>scratch_space</scratch>
-<defaultblocksize>1000</defaultblocksize>
-<cp.parallel.ops>true</cp.parallel.ops>
-<cp.parallel.io>false</cp.parallel.io>
+<sysml.numreducers>10</sysml.numreducers>
+<sysml.scratch>scratch_space</sysml.scratch>
+<sysml.defaultblocksize>1000</sysml.defaultblocksize>
+<sysml.cp.parallel.ops>true</sysml.cp.parallel.ops>
+<sysml.cp.parallel.io>false</sysml.cp.parallel.io>
</root>
http://git-wip-us.apache.org/repos/asf/systemml/blob/d6139d14/src/test/scripts/functions/gdfo/SystemML-config-globalopt.xml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/gdfo/SystemML-config-globalopt.xml b/src/test/scripts/functions/gdfo/SystemML-config-globalopt.xml
index 9cf9316..a59d797 100644
--- a/src/test/scripts/functions/gdfo/SystemML-config-globalopt.xml
+++ b/src/test/scripts/functions/gdfo/SystemML-config-globalopt.xml
@@ -18,43 +18,7 @@
-->
<root>
- <!-- local fs tmp working directory-->
- <localtmpdir>/tmp/systemml</localtmpdir>
-
- <!-- hdfs tmp working directory-->
- <scratch>scratch_space</scratch>
-
- <!-- compiler optimization level, valid values: 0 | 1 | 2 | 3 | 4, default: 2 -->
- <optlevel>4</optlevel>
-
- <!-- default number of reduce tasks per MR job, default: 2 x number of nodes -->
- <numreducers>10</numreducers>
-
- <!-- override jvm reuse flag for specific MR jobs, valid values: true | false -->
- <jvmreuse>false</jvmreuse>
-
- <!-- default block dim for binary block files -->
- <defaultblocksize>1000</defaultblocksize>
-
- <!-- run systemml control program as yarn appmaster, in case of MR1 always falls back to client, please disable for debug mode -->
- <dml.yarn.appmaster>false</dml.yarn.appmaster>
-
- <!-- maximum jvm heap size of the dml yarn appmaster in MB, the requested memory is 1.5x this parameter -->
- <dml.yarn.appmaster.mem>2048</dml.yarn.appmaster.mem>
-
- <!-- maximum jvm heap size of the map/reduce tasks in MB, the requested memory is 1.5x this parameter, negative values ignored -->
- <dml.yarn.mapreduce.mem>2048</dml.yarn.mapreduce.mem>
-
- <!-- yarn application submission queue, relevant for default capacity scheduler -->
- <dml.yarn.app.queue>default</dml.yarn.app.queue>
-
- <!-- enables multi-threaded operations in singlenode control program -->
- <cp.parallel.ops>true</cp.parallel.ops>
-
- <!-- enables multi-threaded read/write in singlenode control program -->
- <cp.parallel.io>true</cp.parallel.io>
-
-
+ <sysml.optlevel>4</sysml.optlevel>
<!-- piggybacked test for custom mapred/mapreduce configurations -->
<mapreduce.task.io.sort.mb>50</mapreduce.task.io.sort.mb>
</root>