You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2018/05/15 03:05:46 UTC
[1/3] systemml git commit: [SYSTEMML-2325] Fix result correctness
bias_add over sparse input
Repository: systemml
Updated Branches:
refs/heads/master cb58c7c16 -> 31580f52d
[SYSTEMML-2325] Fix result correctness bias_add over sparse input
This patch fixes a result correctness issue of the custom bias_add
operation over sparse inputs. This operation used the copy primitive to
take over dense/sparse inputs and subsequently add the bias terms to the
dense output. However, if the input was in sparse format, this led to
switching the output format to sparse such that the bias terms were
added to a discarded dense block and thus lost. We now ensure that the
output is preserved in dense format, which is likely the case for this
additive operation.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/b4dedfeb
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/b4dedfeb
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/b4dedfeb
Branch: refs/heads/master
Commit: b4dedfeb7291311491d8a48f9c4681007cd2366c
Parents: cb58c7c
Author: Matthias Boehm <mb...@gmail.com>
Authored: Mon May 14 18:06:22 2018 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Mon May 14 18:06:22 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/sysml/runtime/matrix/data/LibMatrixDNN.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/b4dedfeb/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDNN.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDNN.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDNN.java
index 17dbdc0..e980246 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDNN.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDNN.java
@@ -312,7 +312,7 @@ public class LibMatrixDNN {
}
else {
// Handles both dense and sparse inputs and copies it to dense output
- outputBlock.copy(input);
+ outputBlock.copy(input, false);
int index = 0;
if(bias.isInSparseFormat())
bias.sparseToDense(); // Since bias is extremely small array
[2/3] systemml git commit: [MINOR][SYSTEMML-2067] New codegen tests
for conv2d and bias_add ops
Posted by mb...@apache.org.
[MINOR][SYSTEMML-2067] New codegen tests for conv2d and bias_add ops
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/94260013
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/94260013
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/94260013
Branch: refs/heads/master
Commit: 94260013acad86bccaf864c97f4f21de3b1b0393
Parents: b4dedfe
Author: Matthias Boehm <mb...@gmail.com>
Authored: Mon May 14 18:45:52 2018 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Mon May 14 18:45:52 2018 -0700
----------------------------------------------------------------------
.../codegen/RowConv2DOperationsTest.java | 136 +++++++++++++++++++
.../scripts/functions/codegen/RowConv2DTest.R | 123 +++++++++++++++++
.../scripts/functions/codegen/RowConv2DTest.dml | 52 +++++++
.../functions/codegen/ZPackageSuite.java | 1 +
4 files changed, 312 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/94260013/src/test/java/org/apache/sysml/test/integration/functions/codegen/RowConv2DOperationsTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/codegen/RowConv2DOperationsTest.java b/src/test/java/org/apache/sysml/test/integration/functions/codegen/RowConv2DOperationsTest.java
new file mode 100644
index 0000000..6910c03
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/codegen/RowConv2DOperationsTest.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sysml.test.integration.functions.codegen;
+
+import java.io.File;
+import java.util.HashMap;
+
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.junit.Test;
+
+public class RowConv2DOperationsTest extends AutomatedTestBase
+{
+ private final static String TEST_NAME1 = "RowConv2DTest";
+ private final static String TEST_DIR = "functions/codegen/";
+ private final static String TEST_CLASS_DIR = TEST_DIR + RowConv2DOperationsTest.class.getSimpleName() + "/";
+
+ private final static String TEST_CONF = "SystemML-config-codegen.xml";
+ private final static File TEST_CONF_FILE = new File(SCRIPT_DIR + TEST_DIR, TEST_CONF);
+
+ private static final double eps = Math.pow(10, -10);
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] {"B"}));
+ }
+
+ @Test
+ public void testConv2DDenseDenseCP() {
+ runConv2DTest(TEST_NAME1, true, 16, 64, 1, 3, 2, 1, 0, false, false, ExecType.CP);
+ }
+
+ @Test
+ public void testConv2DSparseDenseCP() {
+ runConv2DTest(TEST_NAME1, true, 16, 64, 1, 3, 2, 1, 0, true, false, ExecType.CP);
+ }
+
+ @Test
+ public void testConv2DDenseDenseSP() {
+ runConv2DTest(TEST_NAME1, true, 16, 64, 1, 3, 2, 1, 0, false, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testConv2DSparseDenseSP() {
+ runConv2DTest(TEST_NAME1, true, 16, 64, 1, 3, 2, 1, 0, true, false, ExecType.SPARK);
+ }
+
+ public void runConv2DTest(String testname, boolean rewrites, int imgSize, int numImg, int numChannels,
+ int numFilters, int filterSize, int stride, int pad, boolean sparse1, boolean sparse2, ExecType et)
+ {
+ boolean oldFlag = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION;
+ RUNTIME_PLATFORM platformOld = rtplatform;
+ switch( et ) {
+ case MR: rtplatform = RUNTIME_PLATFORM.HADOOP; break;
+ case SPARK: rtplatform = RUNTIME_PLATFORM.SPARK; break;
+ default: rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK; break;
+ }
+
+ boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
+ if( rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK )
+ DMLScript.USE_LOCAL_SPARK_CONFIG = true;
+
+ try
+ {
+ String sparseVal1 = String.valueOf(sparse1).toUpperCase();
+ String sparseVal2 = String.valueOf(sparse2).toUpperCase();
+ TestConfiguration config = getTestConfiguration(testname);
+ loadTestConfiguration(config);
+
+ String HOME = SCRIPT_DIR + TEST_DIR;
+ fullDMLScriptName = HOME + testname + ".dml";
+ programArgs = new String[]{"-explain", "recompile_runtime", "-stats", "-args",
+ String.valueOf(imgSize), String.valueOf(numImg), String.valueOf(numChannels),
+ String.valueOf(numFilters), String.valueOf(filterSize), String.valueOf(stride),
+ String.valueOf(pad), output("B"), sparseVal1, sparseVal2 };
+
+ fullRScriptName = HOME + testname + ".R";
+ rCmd = getRCmd(String.valueOf(imgSize), String.valueOf(numImg), String.valueOf(numChannels),
+ String.valueOf(numFilters), String.valueOf(filterSize), String.valueOf(stride),
+ String.valueOf(pad), expectedDir(), sparseVal1, sparseVal2);
+
+ OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewrites;
+
+ runTest(true, false, null, -1);
+ runRScript(true);
+
+ //compare matrices
+ HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("B");
+ HashMap<CellIndex, Double> rfile = readRMatrixFromFS("B");
+ TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
+ //Assert.assertTrue(heavyHittersContainsSubString("spoofRA")
+ // || heavyHittersContainsSubString("sp_spoofRA"));
+ }
+ finally {
+ rtplatform = platformOld;
+ DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
+ OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = oldFlag;
+ OptimizerUtils.ALLOW_AUTO_VECTORIZATION = true;
+ OptimizerUtils.ALLOW_OPERATOR_FUSION = true;
+ }
+ }
+
+ /**
+ * Override default configuration with custom test configuration to ensure
+ * scratch space and local temporary directory locations are also updated.
+ */
+ @Override
+ protected File getConfigTemplateFile() {
+ // Instrumentation in this test's output log to show custom configuration file used for template.
+ System.out.println("This test case overrides default configuration with " + TEST_CONF_FILE.getPath());
+ return TEST_CONF_FILE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/94260013/src/test/scripts/functions/codegen/RowConv2DTest.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/codegen/RowConv2DTest.R b/src/test/scripts/functions/codegen/RowConv2DTest.R
new file mode 100644
index 0000000..070602d
--- /dev/null
+++ b/src/test/scripts/functions/codegen/RowConv2DTest.R
@@ -0,0 +1,123 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+args <- commandArgs(TRUE)
+library("Matrix")
+imgSize=as.integer(args[1])
+numImg=as.integer(args[2])
+numChannels=as.integer(args[3])
+numFilters=as.integer(args[4])
+filterSize=as.integer(args[5])
+stride=as.integer(args[6])
+pad=as.integer(args[7])
+
+# Assumption: NCHW image format
+x=matrix(seq(1, numImg*numChannels*imgSize*imgSize), numImg, numChannels*imgSize*imgSize, byrow=TRUE)
+w=matrix(seq(1, numFilters*numChannels*filterSize*filterSize), numFilters, numChannels*filterSize*filterSize, byrow=TRUE)
+
+if(as.logical(args[9])) {
+ zero_mask = (x - mean(x)*1.5) > 0
+ x = x * zero_mask
+} else {
+ x = x - mean(x)
+}
+if(as.logical(args[10])) {
+ zero_mask = (w - mean(w)*1.5) > 0
+ w = w * zero_mask
+} else {
+ w = w - mean(w)
+}
+pad_image <- function(img, Hin, Win, padh, padw){
+ C = nrow(img)
+ img_padded = matrix(0, C, (Hin+2*padh)*(Win+2*padw), byrow=TRUE) # zeros
+ for (c in 1:C) {
+ img_slice = matrix(img[c,], Hin, Win, byrow=TRUE) # depth slice C reshaped
+ img_padded_slice = matrix(0, Hin+2*padh, Win+2*padw)
+ img_padded_slice[(padh+1):(padh+Hin), (padw+1):(padw+Win)] = img_slice
+ img_padded[c,] = matrix(t(img_padded_slice), 1, (Hin+2*padh)*(Win+2*padw)) # reshape
+ }
+ img_padded
+}
+
+im2col <- function(img, Hin, Win, Hf, Wf, strideh, stridew) {
+ C = nrow(img)
+ Hout = as.integer((Hin - Hf) / strideh + 1)
+ Wout = as.integer((Win - Wf) / stridew + 1)
+
+ img_cols = matrix(0, C*Hf*Wf, Hout*Wout, byrow=TRUE) # zeros
+ for (hout in 1:Hout) { # all output rows
+ hin = (hout-1) * strideh + 1
+ for (wout in 1:Wout) { # all output columns
+ win = (wout-1) * stridew + 1
+ # Extract a local patch of the input image corresponding spatially to the filter sizes.
+ img_patch = matrix(0, C, Hf*Wf, byrow=TRUE) # zeros
+ for (c in 1:C) { # all channels
+ img_slice = matrix(img[c,], Hin, Win, byrow=TRUE) # reshape
+ img_patch[c,] = matrix(t(img_slice[hin:(hin+Hf-1), win:(win+Wf-1)]), 1, Hf*Wf)
+ }
+ img_cols[,(hout-1)*Wout + wout] = matrix(t(img_patch), C*Hf*Wf, 1) # reshape
+ }
+ }
+ img_cols
+}
+
+conv2d <- function(X, W, C, Hin, Win, Hf, Wf, strideh, stridew, padh, padw) {
+ N = nrow(X)
+ F = nrow(W)
+ Hout = as.integer((Hin + 2 * padh - Hf) / strideh + 1)
+ Wout = as.integer((Win + 2 * padw - Wf) / stridew + 1)
+
+ # Create output volume
+ out = matrix(0, N, F*Hout*Wout, byrow=TRUE)
+
+ # Convolution - im2col implementation
+ for (n in 1:N) { # all examples
+ Xn = matrix(X[n,], C, Hin*Win, byrow=TRUE) # reshape
+
+ # Pad image
+ Xn_padded = pad_image(Xn, Hin, Win, padh, padw) # shape (C, (Hin+2*padh)*(Win+2*padw))
+
+ # Extract local image patches into columns with im2col, of shape (C*Hf*Wf, Hout*Wout)
+ Xn_padded_cols = im2col(Xn_padded, Hin+2*padh, Win+2*padw, Hf, Wf, strideh, stridew)
+
+ # Convolve patches with filters
+ outn = W %*% Xn_padded_cols # shape (F, Hout*Wout)
+ out[n,] = matrix(t(outn), 1, F*Hout*Wout) # reshape
+ }
+
+ out
+}
+
+output = conv2d(x, w, numChannels, imgSize, imgSize, filterSize, filterSize, stride, stride, pad, pad);
+Hout = as.integer((imgSize + 2 * pad - filterSize) / stride + 1)
+Wout = Hout
+
+b=matrix(seq(1, numFilters), numFilters, 1, byrow=TRUE)
+for(k in 0:(numFilters-1)) {
+ for(i in 1:nrow(output)) {
+ start = k*Hout*Hout;
+ for(j in 1:(Hout*Hout)) {
+ output[i,start+j] = output[i,start+j] + b[k+1,1]
+ }
+ }
+}
+
+writeMM(as(output,"CsparseMatrix"), paste(args[8], "B", sep=""))
http://git-wip-us.apache.org/repos/asf/systemml/blob/94260013/src/test/scripts/functions/codegen/RowConv2DTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/codegen/RowConv2DTest.dml b/src/test/scripts/functions/codegen/RowConv2DTest.dml
new file mode 100644
index 0000000..9d51716
--- /dev/null
+++ b/src/test/scripts/functions/codegen/RowConv2DTest.dml
@@ -0,0 +1,52 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+imgSize=$1
+numImg=$2
+numChannels=$3
+numFilters=$4
+filterSize=$5
+stride=$6
+pad=$7
+
+# Assumption: NCHW image format
+x=matrix(seq(1, numImg*numChannels*imgSize*imgSize), rows=numImg, cols=numChannels*imgSize*imgSize)
+w=matrix(seq(1, numFilters*numChannels*filterSize*filterSize), rows=numFilters, cols=numChannels*filterSize*filterSize)
+b=matrix(seq(1, numFilters), rows=numFilters, cols=1)
+
+if($9) {
+ zero_mask = (x - mean(x)*1.5) > 0
+ x = x * zero_mask
+}
+else {
+ x = x - mean(x)
+}
+if($10) {
+ zero_mask = (w - mean(w)*1.5) > 0
+ w = w * zero_mask
+}
+else {
+ w = w - mean(w)
+}
+output = conv2d(x, w, padding=[pad, pad], stride=[stride, stride], input_shape=[numImg, numChannels, imgSize, imgSize], filter_shape=[numFilters, numChannels, filterSize, filterSize], bias=b)
+output = bias_add(output, b)
+
+write(output, $8, format="text")
http://git-wip-us.apache.org/repos/asf/systemml/blob/94260013/src/test_suites/java/org/apache/sysml/test/integration/functions/codegen/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/codegen/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/codegen/ZPackageSuite.java
index 9a78dbc..3857cfa 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/codegen/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/codegen/ZPackageSuite.java
@@ -40,6 +40,7 @@ import org.junit.runners.Suite;
MultiAggTmplTest.class,
OuterProdTmplTest.class,
RowAggTmplTest.class,
+ RowConv2DOperationsTest.class,
RowVectorComparisonTest.class,
SparseSideInputTest.class,
SumProductChainTest.class,
[3/3] systemml git commit: [SYSTEMML-2326] Fix distributed cache
umask issues in local mode
Posted by mb...@apache.org.
[SYSTEMML-2326] Fix distributed cache umask issues in local mode
This patch fixes test issues in local mode for servers with restrictive
umask settings. In such environments the MR distributed cache does not
allow the addition of "hdfs" file (which reside in local file systems
with restricted permissions) to the public cache. We now simple do not
add files to distributed cache in local mode because all relevant jobs
anyway have fallbacks to read from local files in local mode.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/31580f52
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/31580f52
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/31580f52
Branch: refs/heads/master
Commit: 31580f52d318fda5d9ec34297cd6cb4a401b83df
Parents: 9426001
Author: Matthias Boehm <mb...@gmail.com>
Authored: Mon May 14 20:07:09 2018 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Mon May 14 20:07:09 2018 -0700
----------------------------------------------------------------------
.../sysml/runtime/matrix/CSVReblockMR.java | 7 ++++--
.../org/apache/sysml/runtime/matrix/SortMR.java | 23 ++++++++++----------
.../matrix/mapred/MRJobConfiguration.java | 19 ++++++++--------
3 files changed, 26 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/31580f52/src/main/java/org/apache/sysml/runtime/matrix/CSVReblockMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/CSVReblockMR.java b/src/main/java/org/apache/sysml/runtime/matrix/CSVReblockMR.java
index 89c1bee..97379dd 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/CSVReblockMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/CSVReblockMR.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
import org.apache.sysml.runtime.instructions.MRJobInstruction;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
@@ -442,8 +443,10 @@ public class CSVReblockMR
//set unique working dir
MRJobConfiguration.setUniqueWorkingDir(job);
Path cachefile=new Path(counterFile, "part-00000");
- DistributedCache.addCacheFile(cachefile.toUri(), job);
- DistributedCache.createSymlink(job);
+ if( !InfrastructureAnalyzer.isLocalMode(job) ) {
+ DistributedCache.addCacheFile(cachefile.toUri(), job);
+ DistributedCache.createSymlink(job);
+ }
job.set(ROWID_FILE_NAME, cachefile.toString());
RunningJob runjob=JobClient.runJob(job);
http://git-wip-us.apache.org/repos/asf/systemml/blob/31580f52/src/main/java/org/apache/sysml/runtime/matrix/SortMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/SortMR.java b/src/main/java/org/apache/sysml/runtime/matrix/SortMR.java
index d8b7aed..bbd0ef3 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/SortMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/SortMR.java
@@ -239,17 +239,18 @@ public class SortMR
job.setReducerClass(ValueSortReducer.class);
job.setOutputKeyClass(outputInfo.outputKeyClass); //double
job.setOutputValueClass(outputInfo.outputValueClass); //int
- }
- job.setPartitionerClass(TotalOrderPartitioner.class);
-
-
- //setup distributed cache
- DistributedCache.addCacheFile(partitionUri, job);
- DistributedCache.createSymlink(job);
-
- //setup replication factor
- job.setInt(MRConfigurationNames.DFS_REPLICATION, replication);
-
+ }
+ job.setPartitionerClass(TotalOrderPartitioner.class);
+
+ //setup distributed cache
+ if( !InfrastructureAnalyzer.isLocalMode(job) ) {
+ DistributedCache.addCacheFile(partitionUri, job);
+ DistributedCache.createSymlink(job);
+ }
+
+ //setup replication factor
+ job.setInt(MRConfigurationNames.DFS_REPLICATION, replication);
+
//set up custom map/reduce configurations
DMLConfig config = ConfigurationManager.getDMLConfig();
MRJobConfiguration.setupCustomMRConfigurations(job, config);
http://git-wip-us.apache.org/repos/asf/systemml/blob/31580f52/src/main/java/org/apache/sysml/runtime/matrix/mapred/MRJobConfiguration.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MRJobConfiguration.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MRJobConfiguration.java
index 1db292f..7efefbd 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MRJobConfiguration.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MRJobConfiguration.java
@@ -914,11 +914,13 @@ public class MRJobConfiguration
job.set(DISTCACHE_INPUT_PATHS, pathsString);
Path p = null;
- for(String spath : paths) {
- p = new Path(spath);
-
- DistributedCache.addCacheFile(p.toUri(), job);
- DistributedCache.createSymlink(job);
+ if( !InfrastructureAnalyzer.isLocalMode(job) ) {
+ for(String spath : paths) {
+ p = new Path(spath);
+
+ DistributedCache.addCacheFile(p.toUri(), job);
+ DistributedCache.createSymlink(job);
+ }
}
}
@@ -1132,16 +1134,13 @@ public class MRJobConfiguration
outputInfos, inBlockRepresentation, false);
}
- public static String setUpSortPartitionFilename( JobConf job )
- {
+ public static String setUpSortPartitionFilename( JobConf job ) {
String pfname = constructPartitionFilename();
job.set( SORT_PARTITION_FILENAME, pfname );
-
return pfname;
}
- public static String getSortPartitionFilename( JobConf job )
- {
+ public static String getSortPartitionFilename( JobConf job ) {
return job.get( SORT_PARTITION_FILENAME );
}