You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2018/06/22 06:03:19 UTC

systemml git commit: [SYSTEMML-2413] Fix paramserv contention on DAG recompilation

Repository: systemml
Updated Branches:
  refs/heads/master 0e01af524 -> 7027a532d


[SYSTEMML-2413] Fix paramserv contention on DAG recompilation

Closes #789.


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/7027a532
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/7027a532
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/7027a532

Branch: refs/heads/master
Commit: 7027a532dff62304cbde8d18a3f45633c078423b
Parents: 0e01af5
Author: EdgarLGB <gu...@atos.net>
Authored: Thu Jun 21 22:32:07 2018 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu Jun 21 23:02:51 2018 -0700

----------------------------------------------------------------------
 .../controlprogram/FunctionProgramBlock.java    |  2 +-
 .../controlprogram/paramserv/LocalPSWorker.java |  2 +-
 .../controlprogram/paramserv/PSWorker.java      | 17 ++---
 .../controlprogram/paramserv/ParamServer.java   | 37 ++++------
 .../paramserv/ParamservUtils.java               | 76 ++++++++++++--------
 .../cp/ParamservBuiltinCPInstruction.java       | 28 ++++----
 .../paramserv/ParamservRuntimeNegativeTest.java |  4 +-
 .../paramserv/mnist_lenet_paramserv.dml         |  9 ++-
 .../mnist_lenet_paramserv_minimum_version.dml   |  5 +-
 .../paramserv/paramserv-large-parallelism.dml   |  3 +-
 .../paramserv/paramserv-minimum-version.dml     |  3 +-
 .../paramserv/paramserv-nn-asp-batch.dml        |  5 +-
 .../paramserv/paramserv-nn-asp-epoch.dml        |  5 +-
 .../paramserv/paramserv-nn-bsp-batch-dc.dml     |  5 +-
 .../paramserv/paramserv-nn-bsp-batch-dr.dml     |  5 +-
 .../paramserv/paramserv-nn-bsp-batch-drr.dml    |  5 +-
 .../paramserv/paramserv-nn-bsp-batch-or.dml     |  5 +-
 .../paramserv/paramserv-nn-bsp-epoch.dml        |  5 +-
 18 files changed, 114 insertions(+), 107 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/systemml/blob/7027a532/src/main/java/org/apache/sysml/runtime/controlprogram/FunctionProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/FunctionProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/FunctionProgramBlock.java
index 5ea84a7..25b2017 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/FunctionProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/FunctionProgramBlock.java
@@ -110,7 +110,7 @@ public class FunctionProgramBlock extends ProgramBlock
 		}
 		
 		// for each program block
-		try {						
+		try {
 			for (int i=0 ; i < this._childBlocks.size() ; i++) {
 				ec.updateDebugState(i);
 				_childBlocks.get(i).execute(ec);

http://git-wip-us.apache.org/repos/asf/systemml/blob/7027a532/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/LocalPSWorker.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/LocalPSWorker.java b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/LocalPSWorker.java
index beb5e45..4f472ee 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/LocalPSWorker.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/LocalPSWorker.java
@@ -107,7 +107,7 @@ public class LocalPSWorker extends PSWorker implements Callable<Void> {
 	private ListObject updateModel(ListObject globalParams, ListObject gradients, int i, int j, int totalIter) {
 		Timing tUpd = DMLScript.STATISTICS ? new Timing(true) : null;
 
-		globalParams = _ps.updateModel(gradients, globalParams);
+		globalParams = _ps.updateModel(_ec, gradients, globalParams);
 
 		if (DMLScript.STATISTICS)
 			Statistics.accPSLocalModelUpdateTime((long) tUpd.stop());

http://git-wip-us.apache.org/repos/asf/systemml/blob/7027a532/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/PSWorker.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/PSWorker.java b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/PSWorker.java
index f46370d..a76dfec 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/PSWorker.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/PSWorker.java
@@ -19,12 +19,11 @@
 
 package org.apache.sysml.runtime.controlprogram.paramserv;
 
-import static org.apache.sysml.runtime.controlprogram.paramserv.ParamservUtils.UPDATE_FUNC_PREFIX;
+import static org.apache.sysml.runtime.controlprogram.paramserv.ParamservUtils.PS_FUNC_PREFIX;
 
 import java.util.ArrayList;
 import java.util.stream.Collectors;
 
-import org.apache.sysml.parser.DMLProgram;
 import org.apache.sysml.parser.DataIdentifier;
 import org.apache.sysml.parser.Expression;
 import org.apache.sysml.parser.Statement;
@@ -66,14 +65,10 @@ public abstract class PSWorker {
 		_ps = ps;
 
 		// Get the update function
-		String[] keys = DMLProgram.splitFunctionKey(updFunc);
-		String funcName = keys[0];
-		String funcNS = null;
-		if (keys.length == 2) {
-			funcNS = keys[0];
-			funcName = keys[1];
-		}
-		FunctionProgramBlock func = ec.getProgram().getFunctionProgramBlock(funcNS, UPDATE_FUNC_PREFIX + _workerID + "_" + funcName);
+		String[] cfn = ParamservUtils.getCompleteFuncName(updFunc, PS_FUNC_PREFIX);
+		String ns = cfn[0];
+		String fname = cfn[1];
+		FunctionProgramBlock func = ec.getProgram().getFunctionProgramBlock(ns, fname);
 		ArrayList<DataIdentifier> inputs = func.getInputParams();
 		ArrayList<DataIdentifier> outputs = func.getOutputParams();
 		CPOperand[] boundInputs = inputs.stream()
@@ -83,7 +78,7 @@ public abstract class PSWorker {
 			.collect(Collectors.toCollection(ArrayList::new));
 		ArrayList<String> outputNames = outputs.stream().map(DataIdentifier::getName)
 			.collect(Collectors.toCollection(ArrayList::new));
-		_inst = new FunctionCallCPInstruction(funcNS, funcName, boundInputs, inputNames, outputNames, "update function");
+		_inst = new FunctionCallCPInstruction(ns, fname, boundInputs, inputNames, outputNames, "update function");
 
 		// Check the inputs of the update function
 		checkInput(false, inputs, Expression.DataType.MATRIX, Statement.PS_FEATURES);

http://git-wip-us.apache.org/repos/asf/systemml/blob/7027a532/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ParamServer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ParamServer.java b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ParamServer.java
index 0ddfb40..4af72a4 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ParamServer.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ParamServer.java
@@ -19,7 +19,7 @@
 
 package org.apache.sysml.runtime.controlprogram.paramserv;
 
-import static org.apache.sysml.runtime.controlprogram.paramserv.ParamservUtils.AGG_FUNC_PREFIX;
+import static org.apache.sysml.runtime.controlprogram.paramserv.ParamservUtils.PS_FUNC_PREFIX;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -40,14 +40,12 @@ import org.apache.commons.lang3.concurrent.BasicThreadFactory;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.sysml.api.DMLScript;
-import org.apache.sysml.parser.DMLProgram;
 import org.apache.sysml.parser.DataIdentifier;
 import org.apache.sysml.parser.Expression;
 import org.apache.sysml.parser.Statement;
 import org.apache.sysml.runtime.DMLRuntimeException;
 import org.apache.sysml.runtime.controlprogram.FunctionProgramBlock;
 import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
-import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory;
 import org.apache.sysml.runtime.controlprogram.parfor.stat.Timing;
 import org.apache.sysml.runtime.instructions.cp.CPOperand;
 import org.apache.sysml.runtime.instructions.cp.Data;
@@ -101,11 +99,7 @@ public abstract class ParamServer {
 		return _model;
 	}
 
-	public ListObject updateModel(ListObject gradients, ListObject model) {
-		//note: we use a new execution context to allow for concurrent execution of ASP local updates; 
-		//otherwise synchronized on the aggService instance would serialize those
-		ExecutionContext ec = ExecutionContextFactory.createContext(_aggService._ec.getProgram());
-		ec.setVariable(Statement.PS_HYPER_PARAMS, _aggService._ec.getVariable(Statement.PS_HYPER_PARAMS));
+	public ListObject updateModel(ExecutionContext ec, ListObject gradients, ListObject model) {
 		return _aggService.updateModel(ec, gradients, model);
 	}
 
@@ -138,14 +132,10 @@ public abstract class ParamServer {
 			_finishedStates = new boolean[workerNum];
 
 			// Fetch the aggregation function
-			String[] keys = DMLProgram.splitFunctionKey(aggFunc);
-			String funcName = keys[0];
-			String funcNS = null;
-			if (keys.length == 2) {
-				funcNS = keys[0];
-				funcName = keys[1];
-			}
-			FunctionProgramBlock func = _ec.getProgram().getFunctionProgramBlock(funcNS, AGG_FUNC_PREFIX + funcName);
+			String[] cfn = ParamservUtils.getCompleteFuncName(aggFunc, PS_FUNC_PREFIX);
+			String ns = cfn[0];
+			String fname = cfn[1];
+			FunctionProgramBlock func = _ec.getProgram().getFunctionProgramBlock(ns, fname);
 			ArrayList<DataIdentifier> inputs = func.getInputParams();
 			ArrayList<DataIdentifier> outputs = func.getOutputParams();
 
@@ -165,7 +155,7 @@ public abstract class ParamServer {
 				.collect(Collectors.toCollection(ArrayList::new));
 			ArrayList<String> outputNames = outputs.stream().map(DataIdentifier::getName)
 				.collect(Collectors.toCollection(ArrayList::new));
-			_inst = new FunctionCallCPInstruction(funcNS, funcName, boundInputs, inputNames, outputNames, "aggregate function");
+			_inst = new FunctionCallCPInstruction(ns, fname, boundInputs, inputNames, outputNames, "aggregate function");
 		}
 
 		private boolean allFinished() {
@@ -248,16 +238,13 @@ public abstract class ParamServer {
 			return null;
 		}
 
-		/**
-		 * A synchronized service method for updating model with gradients
-		 *
-		 * @param gradients A list object of gradients
-		 * @return A updated list object of model
-		 */
-		private synchronized ListObject updateModel(ListObject gradients, ListObject model) {
+		private ListObject updateModel(ListObject gradients, ListObject model) {
 			return updateModel(_ec, gradients, model);
 		}
-		
+
+		/**
+		 * A service method for updating model with gradients
+		 */
 		private ListObject updateModel(ExecutionContext ec, ListObject gradients, ListObject model) {
 			// Populate the variables table with the gradients and model
 			ec.setVariable(Statement.PS_GRADIENTS, gradients);

http://git-wip-us.apache.org/repos/asf/systemml/blob/7027a532/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ParamservUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ParamservUtils.java b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ParamservUtils.java
index c4a3d98..6c76aa6 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ParamservUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ParamservUtils.java
@@ -26,6 +26,7 @@ import java.util.List;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.sysml.hops.Hop;
 import org.apache.sysml.hops.MultiThreadedHop;
 import org.apache.sysml.hops.OptimizerUtils;
@@ -38,6 +39,7 @@ import org.apache.sysml.runtime.DMLRuntimeException;
 import org.apache.sysml.runtime.controlprogram.ForProgramBlock;
 import org.apache.sysml.runtime.controlprogram.FunctionProgramBlock;
 import org.apache.sysml.runtime.controlprogram.IfProgramBlock;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
 import org.apache.sysml.runtime.controlprogram.ParForProgramBlock;
 import org.apache.sysml.runtime.controlprogram.Program;
 import org.apache.sysml.runtime.controlprogram.ProgramBlock;
@@ -58,8 +60,7 @@ import org.apache.sysml.runtime.matrix.data.OutputInfo;
 
 public class ParamservUtils {
 
-	public static final String UPDATE_FUNC_PREFIX = "_worker_";
-	public static final String AGG_FUNC_PREFIX = "_agg_";
+	public static final String PS_FUNC_PREFIX = "_ps_";
 
 	/**
 	 * Deep copy the list object
@@ -131,7 +132,17 @@ public class ParamservUtils {
 		return permutation;
 	}
 
-	public static ExecutionContext createExecutionContext(ExecutionContext ec, String updFunc, String aggFunc, int workerNum, int k) {
+	public static String[] getCompleteFuncName(String funcName, String prefix) {
+		String[] keys = DMLProgram.splitFunctionKey(funcName);
+		String ns = (keys.length==2) ? keys[0] : null;
+		String name = (keys.length==2) ? keys[1] : keys[0];
+		return StringUtils.isEmpty(prefix) ? 
+			new String[]{ns, name} : new String[]{ns, name};
+	}
+
+	public static List<ExecutionContext> createExecutionContexts(ExecutionContext ec, LocalVariableMap varsMap,
+		String updFunc, String aggFunc, int workerNum, int k) {
+
 		FunctionProgramBlock updPB = getFunctionBlock(ec, updFunc);
 		FunctionProgramBlock aggPB = getFunctionBlock(ec, aggFunc);
 
@@ -142,27 +153,40 @@ public class ParamservUtils {
 		// 2. Recompile the imported function blocks
 		prog.getFunctionProgramBlocks().forEach((fname, fvalue) -> recompileProgramBlocks(k, fvalue.getChildBlocks()));
 
-		// Copy function for workers
-		IntStream.range(0, workerNum).forEach(i -> copyFunction(updFunc, updPB, prog, UPDATE_FUNC_PREFIX + i + "_"));
+		// 3. Copy function for workers
+		List<ExecutionContext> workerECs = IntStream.range(0, workerNum)
+			.mapToObj(i -> {
+				FunctionProgramBlock newUpdFunc = copyFunction(updFunc, updPB);
+				FunctionProgramBlock newAggFunc = copyFunction(aggFunc, aggPB);
+				Program newProg = new Program();
+				putFunction(newProg, newUpdFunc);
+				putFunction(newProg, newAggFunc);
+				return ExecutionContextFactory.createContext(new LocalVariableMap(varsMap), newProg);
+			})
+			.collect(Collectors.toList());
 
-		// Copy function for agg service
-		copyFunction(aggFunc, aggPB, prog, AGG_FUNC_PREFIX);
+		// 4. Copy function for agg service
+		FunctionProgramBlock newAggFunc = copyFunction(aggFunc, aggPB);
+		Program newProg = new Program();
+		putFunction(newProg, newAggFunc);
+		ExecutionContext aggEC = ExecutionContextFactory.createContext(new LocalVariableMap(varsMap), newProg);
 
-		return ExecutionContextFactory.createContext(prog);
+		List<ExecutionContext> result = new ArrayList<>(workerECs);
+		result.add(aggEC);
+		return result;
 	}
 
-	private static void copyFunction(String funcName, FunctionProgramBlock updPB, Program prog, String prefix) {
-		String[] keys = DMLProgram.splitFunctionKey(funcName);
-		String namespace = null;
-		String func = keys[0];
-		if (keys.length == 2) {
-			namespace = keys[0];
-			func = keys[1];
-		}
-		FunctionProgramBlock copiedFunc = ProgramConverter
-			.createDeepCopyFunctionProgramBlock(updPB, new HashSet<>(), new HashSet<>());
-		String fnameNew = prefix + func;
-		prog.addFunctionProgramBlock(namespace, fnameNew, copiedFunc);
+	private static FunctionProgramBlock copyFunction(String funcName, FunctionProgramBlock fpb) {
+		FunctionProgramBlock copiedFunc = ProgramConverter.createDeepCopyFunctionProgramBlock(fpb, new HashSet<>(), new HashSet<>());
+		String[] cfn = getCompleteFuncName(funcName, ParamservUtils.PS_FUNC_PREFIX);
+		copiedFunc._namespace = cfn[0];
+		copiedFunc._functionName = cfn[1];
+		return copiedFunc;
+	}
+
+	private static void putFunction(Program prog, FunctionProgramBlock fpb) {
+		prog.addFunctionProgramBlock(fpb._namespace, fpb._functionName, fpb);
+		prog.addProgramBlock(fpb);
 	}
 
 	private static void recompileProgramBlocks(int k, ArrayList<ProgramBlock> pbs) {
@@ -229,13 +253,9 @@ public class ParamservUtils {
 
 
 	private static FunctionProgramBlock getFunctionBlock(ExecutionContext ec, String funcName) {
-		String[] keys = DMLProgram.splitFunctionKey(funcName);
-		String namespace = null;
-		String func = keys[0];
-		if (keys.length == 2) {
-			namespace = keys[0];
-			func = keys[1];
-		}
-		return ec.getProgram().getFunctionProgramBlock(namespace, func);
+		String[] cfn = getCompleteFuncName(funcName, null);
+		String ns = cfn[0];
+		String fname = cfn[1];
+		return ec.getProgram().getFunctionProgramBlock(ns, fname);
 	}
 }

http://git-wip-us.apache.org/repos/asf/systemml/blob/7027a532/src/main/java/org/apache/sysml/runtime/instructions/cp/ParamservBuiltinCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ParamservBuiltinCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ParamservBuiltinCPInstruction.java
index d2c0edd..be80127 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ParamservBuiltinCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ParamservBuiltinCPInstruction.java
@@ -56,10 +56,8 @@ import org.apache.log4j.Logger;
 import org.apache.sysml.api.DMLScript;
 import org.apache.sysml.runtime.DMLRuntimeException;
 import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
-import org.apache.sysml.runtime.controlprogram.Program;
 import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
 import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
-import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory;
 import org.apache.sysml.runtime.controlprogram.paramserv.DataPartitioner;
 import org.apache.sysml.runtime.controlprogram.paramserv.DataPartitionerDC;
 import org.apache.sysml.runtime.controlprogram.paramserv.DataPartitionerDR;
@@ -112,13 +110,15 @@ public class ParamservBuiltinCPInstruction extends ParameterizedBuiltinCPInstruc
 		int k = getParLevel(workerNum);
 
 		// Get the compiled execution context
-		ExecutionContext newEC = ParamservUtils.createExecutionContext(ec, updFunc, aggFunc, workerNum, k);
+		// Create workers' execution context
+		LocalVariableMap newVarsMap = createVarsMap(ec);
+		List<ExecutionContext> newECs = ParamservUtils.createExecutionContexts(ec, newVarsMap, updFunc, aggFunc, workerNum, k);
 
 		// Create workers' execution context
-		List<ExecutionContext> workerECs = createExecutionContext(workerNum, ec, newEC.getProgram());
+		List<ExecutionContext> workerECs = newECs.subList(0, newECs.size() - 1);
 
 		// Create the agg service's execution context
-		ExecutionContext aggServiceEC = createExecutionContext(1, ec, newEC.getProgram()).get(0);
+		ExecutionContext aggServiceEC = newECs.get(newECs.size() - 1);
 
 		PSFrequency freq = getFrequency();
 		PSUpdateType updateType = getUpdateType();
@@ -165,16 +165,14 @@ public class ParamservBuiltinCPInstruction extends ParameterizedBuiltinCPInstruc
 		}
 	}
 
-	private List<ExecutionContext> createExecutionContext(int size, ExecutionContext ec, Program program) {
-		return IntStream.range(0, size).mapToObj(i -> {
-			// Put the hyperparam into the variables table
-			LocalVariableMap varsMap = new LocalVariableMap();
-			ListObject hyperParams = getHyperParams(ec);
-			if (hyperParams != null) {
-				varsMap.put(PS_HYPER_PARAMS, hyperParams);
-			}
-			return ExecutionContextFactory.createContext(varsMap, program);
-		}).collect(Collectors.toList());
+	private LocalVariableMap createVarsMap(ExecutionContext ec) {
+		// Put the hyperparam into the variables table
+		LocalVariableMap varsMap = new LocalVariableMap();
+		ListObject hyperParams = getHyperParams(ec);
+		if (hyperParams != null) {
+			varsMap.put(PS_HYPER_PARAMS, hyperParams);
+		}
+		return varsMap;
 	}
 
 	private PSModeType getPSMode() {

http://git-wip-us.apache.org/repos/asf/systemml/blob/7027a532/src/test/java/org/apache/sysml/test/integration/functions/paramserv/ParamservRuntimeNegativeTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/paramserv/ParamservRuntimeNegativeTest.java b/src/test/java/org/apache/sysml/test/integration/functions/paramserv/ParamservRuntimeNegativeTest.java
index 7238cf9..a81fd36 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/paramserv/ParamservRuntimeNegativeTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/paramserv/ParamservRuntimeNegativeTest.java
@@ -44,12 +44,12 @@ public class ParamservRuntimeNegativeTest extends AutomatedTestBase {
 
 	@Test
 	public void testParamservWorkerFailed() {
-		runDMLTest(TEST_NAME1, "Invalid lookup by name in unnamed list: worker_err.");
+		runDMLTest(TEST_NAME1, "Invalid indexing by name in unnamed list: worker_err.");
 	}
 
 	@Test
 	public void testParamservAggServiceFailed() {
-		runDMLTest(TEST_NAME2, "Invalid lookup by name in unnamed list: agg_service_err");
+		runDMLTest(TEST_NAME2, "Invalid indexing by name in unnamed list: agg_service_err");
 	}
 
 	@Test

http://git-wip-us.apache.org/repos/asf/systemml/blob/7027a532/src/test/scripts/functions/paramserv/mnist_lenet_paramserv.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/paramserv/mnist_lenet_paramserv.dml b/src/test/scripts/functions/paramserv/mnist_lenet_paramserv.dml
index a10c846..acafc88 100644
--- a/src/test/scripts/functions/paramserv/mnist_lenet_paramserv.dml
+++ b/src/test/scripts/functions/paramserv/mnist_lenet_paramserv.dml
@@ -36,7 +36,7 @@ source("nn/optim/sgd_nesterov.dml") as sgd_nesterov
 train = function(matrix[double] X, matrix[double] Y,
                  matrix[double] X_val, matrix[double] Y_val,
                  int C, int Hin, int Win, int epochs, int workers,
-                 string utype, string freq, string scheme)
+                 string utype, string freq, int batchsize, string scheme)
     return (matrix[double] W1, matrix[double] b1,
             matrix[double] W2, matrix[double] b2,
             matrix[double] W3, matrix[double] b3,
@@ -108,7 +108,7 @@ train = function(matrix[double] X, matrix[double] Y,
   params = list(lr=lr, mu=mu, decay=decay, C=C, Hin=Hin, Win=Win, Hf=Hf, Wf=Wf, stride=stride, pad=pad, lambda=lambda, F1=F1, F2=F2, N3=N3)
 
   # Use paramserv function
-  modelList2 = paramserv(model=modelList, features=X, labels=Y, val_features=X_val, val_labels=Y_val, upd="./src/test/scripts/functions/paramserv/mnist_lenet_paramserv.dml::gradients", agg="./src/test/scripts/functions/paramserv/mnist_lenet_paramserv.dml::aggregation", mode="LOCAL", utype=utype, freq=freq, epochs=epochs, batchsize=64, k=workers, scheme=scheme, hyperparams=params, checkpointing="NONE")
+  modelList2 = paramserv(model=modelList, features=X, labels=Y, val_features=X_val, val_labels=Y_val, upd="./src/test/scripts/functions/paramserv/mnist_lenet_paramserv.dml::gradients", agg="./src/test/scripts/functions/paramserv/mnist_lenet_paramserv.dml::aggregation", mode="LOCAL", utype=utype, freq=freq, epochs=epochs, batchsize=batchsize, k=workers, scheme=scheme, hyperparams=params, checkpointing="NONE")
 
   W1 = as.matrix(modelList2["W1"])
   b1 = as.matrix(modelList2["b1"])
@@ -256,7 +256,7 @@ aggregation = function(list[unknown] model,
      modelResult = list(W1=W1, b1=b1, W2=W2, b2=b2, W3=W3, b3=b3, W4=W4, b4=b4, vW1=vW1, vW2=vW2, vW3=vW3, vW4=vW4, vb1=vb1, vb2=vb2, vb3=vb3, vb4=vb4)
    }
 
-predict = function(matrix[double] X, int C, int Hin, int Win,
+predict = function(matrix[double] X, int C, int Hin, int Win, int batch_size,
                    matrix[double] W1, matrix[double] b1,
                    matrix[double] W2, matrix[double] b2,
                    matrix[double] W3, matrix[double] b3,
@@ -302,9 +302,8 @@ predict = function(matrix[double] X, int C, int Hin, int Win,
 
   # Compute predictions over mini-batches
   probs = matrix(0, rows=N, cols=K)
-  batch_size = 64
   iters = ceil(N / batch_size)
-  for(i in 1:iters) {
+  parfor(i in 1:iters, check=0) {
     # Get next batch
     beg = ((i-1) * batch_size) %% N + 1
     end = min(N, beg + batch_size - 1)

http://git-wip-us.apache.org/repos/asf/systemml/blob/7027a532/src/test/scripts/functions/paramserv/mnist_lenet_paramserv_minimum_version.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/paramserv/mnist_lenet_paramserv_minimum_version.dml b/src/test/scripts/functions/paramserv/mnist_lenet_paramserv_minimum_version.dml
index d02e5d6..aeec3df 100644
--- a/src/test/scripts/functions/paramserv/mnist_lenet_paramserv_minimum_version.dml
+++ b/src/test/scripts/functions/paramserv/mnist_lenet_paramserv_minimum_version.dml
@@ -249,7 +249,7 @@ aggregation = function(list[unknown] model,
      modelResult = list(W1=W1, b1=b1, W2=W2, b2=b2, W3=W3, b3=b3, W4=W4, b4=b4, vW1=vW1, vW2=vW2, vW3=vW3, vW4=vW4, vb1=vb1, vb2=vb2, vb3=vb3, vb4=vb4)
    }
 
-predict = function(matrix[double] X, int C, int Hin, int Win,
+predict = function(matrix[double] X, int C, int Hin, int Win, int batch_size,
                    matrix[double] W1, matrix[double] b1,
                    matrix[double] W2, matrix[double] b2,
                    matrix[double] W3, matrix[double] b3,
@@ -295,9 +295,8 @@ predict = function(matrix[double] X, int C, int Hin, int Win,
 
   # Compute predictions over mini-batches
   probs = matrix(0, rows=N, cols=K)
-  batch_size = 64
   iters = ceil(N / batch_size)
-  for(i in 1:iters) {
+  parfor(i in 1:iters, check=0) {
     # Get next batch
     beg = ((i-1) * batch_size) %% N + 1
     end = min(N, beg + batch_size - 1)

http://git-wip-us.apache.org/repos/asf/systemml/blob/7027a532/src/test/scripts/functions/paramserv/paramserv-large-parallelism.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/paramserv/paramserv-large-parallelism.dml b/src/test/scripts/functions/paramserv/paramserv-large-parallelism.dml
index 4e2d2b7..de29a04 100644
--- a/src/test/scripts/functions/paramserv/paramserv-large-parallelism.dml
+++ b/src/test/scripts/functions/paramserv/paramserv-large-parallelism.dml
@@ -39,12 +39,13 @@ Y_val = labels[1:val_size,]
 # Arguments
 epochs = 10
 workers = 10
+batchsize = 64
 
 # Train
 [W1, b1, W2, b2, W3, b3, W4, b4] = mnist_lenet::train(X, Y, X_val, Y_val, C, Hin, Win, epochs, workers)
 
 # Compute validation loss & accuracy
-probs_val = mnist_lenet::predict(X_val, C, Hin, Win, W1, b1, W2, b2, W3, b3, W4, b4)
+probs_val = mnist_lenet::predict(X_val, C, Hin, Win, batchsize, W1, b1, W2, b2, W3, b3, W4, b4)
 loss_val = cross_entropy_loss::forward(probs_val, Y_val)
 accuracy_val = mean(rowIndexMax(probs_val) == rowIndexMax(Y_val))
 

http://git-wip-us.apache.org/repos/asf/systemml/blob/7027a532/src/test/scripts/functions/paramserv/paramserv-minimum-version.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/paramserv/paramserv-minimum-version.dml b/src/test/scripts/functions/paramserv/paramserv-minimum-version.dml
index 4d23b8c..a537fe9 100644
--- a/src/test/scripts/functions/paramserv/paramserv-minimum-version.dml
+++ b/src/test/scripts/functions/paramserv/paramserv-minimum-version.dml
@@ -39,12 +39,13 @@ Y_val = labels[1:val_size,]
 # Arguments
 epochs = 10
 workers = 2
+batchsize = 64
 
 # Train
 [W1, b1, W2, b2, W3, b3, W4, b4] = mnist_lenet::train(X, Y, X_val, Y_val, C, Hin, Win, epochs, workers)
 
 # Compute validation loss & accuracy
-probs_val = mnist_lenet::predict(X_val, C, Hin, Win, W1, b1, W2, b2, W3, b3, W4, b4)
+probs_val = mnist_lenet::predict(X_val, C, Hin, Win, batchsize, W1, b1, W2, b2, W3, b3, W4, b4)
 loss_val = cross_entropy_loss::forward(probs_val, Y_val)
 accuracy_val = mean(rowIndexMax(probs_val) == rowIndexMax(Y_val))
 

http://git-wip-us.apache.org/repos/asf/systemml/blob/7027a532/src/test/scripts/functions/paramserv/paramserv-nn-asp-batch.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/paramserv/paramserv-nn-asp-batch.dml b/src/test/scripts/functions/paramserv/paramserv-nn-asp-batch.dml
index baef6ee..2279d58 100644
--- a/src/test/scripts/functions/paramserv/paramserv-nn-asp-batch.dml
+++ b/src/test/scripts/functions/paramserv/paramserv-nn-asp-batch.dml
@@ -39,12 +39,13 @@ Y_val = labels[1:val_size,]
 # Arguments
 epochs = 10
 workers = 2
+batchsize = 32
 
 # Train
-[W1, b1, W2, b2, W3, b3, W4, b4] = mnist_lenet::train(X, Y, X_val, Y_val, C, Hin, Win, epochs, workers, "ASP", "BATCH", "DISJOINT_CONTIGUOUS")
+[W1, b1, W2, b2, W3, b3, W4, b4] = mnist_lenet::train(X, Y, X_val, Y_val, C, Hin, Win, epochs, workers, "ASP", "BATCH", batchsize,"DISJOINT_CONTIGUOUS")
 
 # Compute validation loss & accuracy
-probs_val = mnist_lenet::predict(X_val, C, Hin, Win, W1, b1, W2, b2, W3, b3, W4, b4)
+probs_val = mnist_lenet::predict(X_val, C, Hin, Win, batchsize, W1, b1, W2, b2, W3, b3, W4, b4)
 loss_val = cross_entropy_loss::forward(probs_val, Y_val)
 accuracy_val = mean(rowIndexMax(probs_val) == rowIndexMax(Y_val))
 

http://git-wip-us.apache.org/repos/asf/systemml/blob/7027a532/src/test/scripts/functions/paramserv/paramserv-nn-asp-epoch.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/paramserv/paramserv-nn-asp-epoch.dml b/src/test/scripts/functions/paramserv/paramserv-nn-asp-epoch.dml
index 860f53f..1824083 100644
--- a/src/test/scripts/functions/paramserv/paramserv-nn-asp-epoch.dml
+++ b/src/test/scripts/functions/paramserv/paramserv-nn-asp-epoch.dml
@@ -39,12 +39,13 @@ Y_val = labels[1:val_size,]
 # Arguments
 epochs = 10
 workers = 2
+batchsize = 32
 
 # Train
-[W1, b1, W2, b2, W3, b3, W4, b4] = mnist_lenet::train(X, Y, X_val, Y_val, C, Hin, Win, epochs, workers, "ASP", "EPOCH", "DISJOINT_CONTIGUOUS")
+[W1, b1, W2, b2, W3, b3, W4, b4] = mnist_lenet::train(X, Y, X_val, Y_val, C, Hin, Win, epochs, workers, "ASP", "EPOCH", batchsize, "DISJOINT_CONTIGUOUS")
 
 # Compute validation loss & accuracy
-probs_val = mnist_lenet::predict(X_val, C, Hin, Win, W1, b1, W2, b2, W3, b3, W4, b4)
+probs_val = mnist_lenet::predict(X_val, C, Hin, Win, batchsize, W1, b1, W2, b2, W3, b3, W4, b4)
 loss_val = cross_entropy_loss::forward(probs_val, Y_val)
 accuracy_val = mean(rowIndexMax(probs_val) == rowIndexMax(Y_val))
 

http://git-wip-us.apache.org/repos/asf/systemml/blob/7027a532/src/test/scripts/functions/paramserv/paramserv-nn-bsp-batch-dc.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/paramserv/paramserv-nn-bsp-batch-dc.dml b/src/test/scripts/functions/paramserv/paramserv-nn-bsp-batch-dc.dml
index dcbd2dd..2e09de4 100644
--- a/src/test/scripts/functions/paramserv/paramserv-nn-bsp-batch-dc.dml
+++ b/src/test/scripts/functions/paramserv/paramserv-nn-bsp-batch-dc.dml
@@ -39,12 +39,13 @@ Y_val = labels[1:val_size,]
 # Arguments
 epochs = 10
 workers = 2
+batchsize = 32
 
 # Train
-[W1, b1, W2, b2, W3, b3, W4, b4] = mnist_lenet::train(X, Y, X_val, Y_val, C, Hin, Win, epochs, workers, "BSP", "BATCH", "DISJOINT_CONTIGUOUS")
+[W1, b1, W2, b2, W3, b3, W4, b4] = mnist_lenet::train(X, Y, X_val, Y_val, C, Hin, Win, epochs, workers, "BSP", "BATCH", batchsize, "DISJOINT_CONTIGUOUS")
 
 # Compute validation loss & accuracy
-probs_val = mnist_lenet::predict(X_val, C, Hin, Win, W1, b1, W2, b2, W3, b3, W4, b4)
+probs_val = mnist_lenet::predict(X_val, C, Hin, Win, batchsize, W1, b1, W2, b2, W3, b3, W4, b4)
 loss_val = cross_entropy_loss::forward(probs_val, Y_val)
 accuracy_val = mean(rowIndexMax(probs_val) == rowIndexMax(Y_val))
 

http://git-wip-us.apache.org/repos/asf/systemml/blob/7027a532/src/test/scripts/functions/paramserv/paramserv-nn-bsp-batch-dr.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/paramserv/paramserv-nn-bsp-batch-dr.dml b/src/test/scripts/functions/paramserv/paramserv-nn-bsp-batch-dr.dml
index 96fe734..8444952 100644
--- a/src/test/scripts/functions/paramserv/paramserv-nn-bsp-batch-dr.dml
+++ b/src/test/scripts/functions/paramserv/paramserv-nn-bsp-batch-dr.dml
@@ -39,12 +39,13 @@ Y_val = labels[1:val_size,]
 # Arguments
 epochs = 10
 workers = 2
+batchsize = 32
 
 # Train
-[W1, b1, W2, b2, W3, b3, W4, b4] = mnist_lenet::train(X, Y, X_val, Y_val, C, Hin, Win, epochs, workers, "BSP", "BATCH", "DISJOINT_RANDOM")
+[W1, b1, W2, b2, W3, b3, W4, b4] = mnist_lenet::train(X, Y, X_val, Y_val, C, Hin, Win, epochs, workers, "BSP", "BATCH", batchsize, "DISJOINT_RANDOM")
 
 # Compute validation loss & accuracy
-probs_val = mnist_lenet::predict(X_val, C, Hin, Win, W1, b1, W2, b2, W3, b3, W4, b4)
+probs_val = mnist_lenet::predict(X_val, C, Hin, Win, batchsize, W1, b1, W2, b2, W3, b3, W4, b4)
 loss_val = cross_entropy_loss::forward(probs_val, Y_val)
 accuracy_val = mean(rowIndexMax(probs_val) == rowIndexMax(Y_val))
 

http://git-wip-us.apache.org/repos/asf/systemml/blob/7027a532/src/test/scripts/functions/paramserv/paramserv-nn-bsp-batch-drr.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/paramserv/paramserv-nn-bsp-batch-drr.dml b/src/test/scripts/functions/paramserv/paramserv-nn-bsp-batch-drr.dml
index e97dbff..ccb7ffc 100644
--- a/src/test/scripts/functions/paramserv/paramserv-nn-bsp-batch-drr.dml
+++ b/src/test/scripts/functions/paramserv/paramserv-nn-bsp-batch-drr.dml
@@ -39,12 +39,13 @@ Y_val = labels[1:val_size,]
 # Arguments
 epochs = 10
 workers = 4
+batchsize = 32
 
 # Train
-[W1, b1, W2, b2, W3, b3, W4, b4] = mnist_lenet::train(X, Y, X_val, Y_val, C, Hin, Win, epochs, workers, "BSP", "BATCH", "DISJOINT_ROUND_ROBIN")
+[W1, b1, W2, b2, W3, b3, W4, b4] = mnist_lenet::train(X, Y, X_val, Y_val, C, Hin, Win, epochs, workers, "BSP", "BATCH", batchsize, "DISJOINT_ROUND_ROBIN")
 
 # Compute validation loss & accuracy
-probs_val = mnist_lenet::predict(X_val, C, Hin, Win, W1, b1, W2, b2, W3, b3, W4, b4)
+probs_val = mnist_lenet::predict(X_val, C, Hin, Win, batchsize, W1, b1, W2, b2, W3, b3, W4, b4)
 loss_val = cross_entropy_loss::forward(probs_val, Y_val)
 accuracy_val = mean(rowIndexMax(probs_val) == rowIndexMax(Y_val))
 

http://git-wip-us.apache.org/repos/asf/systemml/blob/7027a532/src/test/scripts/functions/paramserv/paramserv-nn-bsp-batch-or.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/paramserv/paramserv-nn-bsp-batch-or.dml b/src/test/scripts/functions/paramserv/paramserv-nn-bsp-batch-or.dml
index a2e95d3..4afc56b 100644
--- a/src/test/scripts/functions/paramserv/paramserv-nn-bsp-batch-or.dml
+++ b/src/test/scripts/functions/paramserv/paramserv-nn-bsp-batch-or.dml
@@ -39,12 +39,13 @@ Y_val = labels[1:val_size,]
 # Arguments
 epochs = 10
 workers = 2
+batchsize = 32
 
 # Train
-[W1, b1, W2, b2, W3, b3, W4, b4] = mnist_lenet::train(X, Y, X_val, Y_val, C, Hin, Win, epochs, workers, "BSP", "BATCH", "OVERLAP_RESHUFFLE")
+[W1, b1, W2, b2, W3, b3, W4, b4] = mnist_lenet::train(X, Y, X_val, Y_val, C, Hin, Win, epochs, workers, "BSP", "BATCH", batchsize, "OVERLAP_RESHUFFLE")
 
 # Compute validation loss & accuracy
-probs_val = mnist_lenet::predict(X_val, C, Hin, Win, W1, b1, W2, b2, W3, b3, W4, b4)
+probs_val = mnist_lenet::predict(X_val, C, Hin, Win, batchsize, W1, b1, W2, b2, W3, b3, W4, b4)
 loss_val = cross_entropy_loss::forward(probs_val, Y_val)
 accuracy_val = mean(rowIndexMax(probs_val) == rowIndexMax(Y_val))
 

http://git-wip-us.apache.org/repos/asf/systemml/blob/7027a532/src/test/scripts/functions/paramserv/paramserv-nn-bsp-epoch.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/paramserv/paramserv-nn-bsp-epoch.dml b/src/test/scripts/functions/paramserv/paramserv-nn-bsp-epoch.dml
index 25d5f48..c542286 100644
--- a/src/test/scripts/functions/paramserv/paramserv-nn-bsp-epoch.dml
+++ b/src/test/scripts/functions/paramserv/paramserv-nn-bsp-epoch.dml
@@ -39,12 +39,13 @@ Y_val = labels[1:val_size,]
 # Arguments
 epochs = 10
 workers = 2
+batchsize = 32
 
 # Train
-[W1, b1, W2, b2, W3, b3, W4, b4] = mnist_lenet::train(X, Y, X_val, Y_val, C, Hin, Win, epochs, workers, "BSP", "EPOCH", "DISJOINT_CONTIGUOUS")
+[W1, b1, W2, b2, W3, b3, W4, b4] = mnist_lenet::train(X, Y, X_val, Y_val, C, Hin, Win, epochs, workers, "BSP", "EPOCH", batchsize,"DISJOINT_CONTIGUOUS")
 
 # Compute validation loss & accuracy
-probs_val = mnist_lenet::predict(X_val, C, Hin, Win, W1, b1, W2, b2, W3, b3, W4, b4)
+probs_val = mnist_lenet::predict(X_val, C, Hin, Win, batchsize, W1, b1, W2, b2, W3, b3, W4, b4)
 loss_val = cross_entropy_loss::forward(probs_val, Y_val)
 accuracy_val = mean(rowIndexMax(probs_val) == rowIndexMax(Y_val))