You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2018/07/13 06:51:32 UTC

[2/2] systemml git commit: [SYSTEMML-2418] New paramserv distributed spark data partitioners

[SYSTEMML-2418] New paramserv distributed spark data partitioners

Closes #793.


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/8def2040
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/8def2040
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/8def2040

Branch: refs/heads/master
Commit: 8def2040be8e9704c8ee8083e8b949ffc0d74927
Parents: 2f4bce9
Author: EdgarLGB <gu...@atos.net>
Authored: Thu Jul 12 22:25:44 2018 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu Jul 12 23:48:38 2018 -0700

----------------------------------------------------------------------
 .../context/SparkExecutionContext.java          |   9 +-
 .../controlprogram/paramserv/DCScheme.java      |  61 ++++++
 .../controlprogram/paramserv/DRRScheme.java     |  57 ++++++
 .../controlprogram/paramserv/DRScheme.java      |  62 ++++++
 .../paramserv/DataPartitionScheme.java          |  40 ++++
 .../paramserv/DataPartitioner.java              |  39 ++--
 .../paramserv/DataPartitionerDC.java            |  55 ------
 .../paramserv/DataPartitionerDR.java            |  64 -------
 .../paramserv/DataPartitionerDRR.java           |  61 ------
 .../paramserv/DataPartitionerOR.java            |  62 ------
 .../controlprogram/paramserv/LocalPSWorker.java |   1 -
 .../controlprogram/paramserv/ORScheme.java      |  61 ++++++
 .../paramserv/ParamservUtils.java               | 141 ++++++++++++--
 .../paramserv/spark/DCSparkScheme.java          |  47 +++++
 .../paramserv/spark/DRRSparkScheme.java         |  45 +++++
 .../paramserv/spark/DRSparkScheme.java          |  69 +++++++
 .../spark/DataPartitionSparkScheme.java         |  76 ++++++++
 .../spark/DataPartitionerSparkAggregator.java   |  66 +++++++
 .../spark/DataPartitionerSparkMapper.java       |  70 +++++++
 .../paramserv/spark/ORSparkScheme.java          |  60 ++++++
 .../paramserv/spark/SparkDataPartitioner.java   | 106 ++++++++++
 .../paramserv/spark/SparkPSWorker.java          |  48 +++++
 .../cp/ParamservBuiltinCPInstruction.java       | 101 +++++-----
 .../paramserv/BaseDataPartitionerTest.java      |  80 ++++++++
 .../paramserv/DataPartitionerTest.java          | 192 -------------------
 .../paramserv/LocalDataPartitionerTest.java     | 135 +++++++++++++
 .../paramserv/SparkDataPartitionerTest.java     |  97 ++++++++++
 .../functions/paramserv/ZPackageSuite.java      |   3 +-
 28 files changed, 1396 insertions(+), 512 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/systemml/blob/8def2040/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
index a3c7677..f8d2345 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
@@ -546,7 +546,7 @@ public class SparkExecutionContext extends ExecutionContext
 	}
 
 	@SuppressWarnings("unchecked")
-	public PartitionedBroadcast<MatrixBlock> getBroadcastForVariable(String varname) {
+	public PartitionedBroadcast<MatrixBlock> getBroadcastForMatrixObject(MatrixObject mo) {
 		//NOTE: The memory consumption of this method is the in-memory size of the 
 		//matrix object plus the partitioned size in 1k-1k blocks. Since the call
 		//to broadcast happens after the matrix object has been released, the memory
@@ -557,8 +557,6 @@ public class SparkExecutionContext extends ExecutionContext
 		
 		long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
 
-		MatrixObject mo = getMatrixObject(varname);
-
 		PartitionedBroadcast<MatrixBlock> bret = null;
 
 		//reuse existing broadcast handle
@@ -613,6 +611,11 @@ public class SparkExecutionContext extends ExecutionContext
 		return bret;
 	}
 
+	public PartitionedBroadcast<MatrixBlock> getBroadcastForVariable(String varname) {
+		MatrixObject mo = getMatrixObject(varname);
+		return getBroadcastForMatrixObject(mo);
+	}
+
 	@SuppressWarnings("unchecked")
 	public PartitionedBroadcast<FrameBlock> getBroadcastForFrameVariable(String varname) {
 		long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;

http://git-wip-us.apache.org/repos/asf/systemml/blob/8def2040/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DCScheme.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DCScheme.java b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DCScheme.java
new file mode 100644
index 0000000..00aaa21
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DCScheme.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.controlprogram.paramserv;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+
+/**
+ * Disjoint_Contiguous data partitioner:
+ *
+ * for each worker, use a right indexing
+ * operation X[beg:end,] to obtain contiguous,
+ * non-overlapping partitions of rows.
+ */
+public class DCScheme extends DataPartitionScheme {
+
+	public static List<MatrixBlock> partition(int k, MatrixBlock mb) {
+		List<MatrixBlock> list = new ArrayList<>();
+		long stepSize = (long) Math.ceil((double) mb.getNumRows() / k);
+		long begin = 1;
+		while (begin < mb.getNumRows()) {
+			long end = Math.min(begin - 1 + stepSize, mb.getNumRows());
+			MatrixBlock pmo = ParamservUtils.sliceMatrixBlock(mb, begin, end);
+			list.add(pmo);
+			begin = end + 1;
+		}
+		return list;
+	}
+
+	private List<MatrixObject> doPartitioning(int k, MatrixBlock mb) {
+		return partition(k, mb).stream().map(ParamservUtils::newMatrixObject).collect(Collectors.toList());
+	}
+
+	@Override
+	public Result doPartitioning(int workersNum, MatrixBlock features, MatrixBlock labels) {
+		List<MatrixObject> pfs = doPartitioning(workersNum, features);
+		List<MatrixObject> pls = doPartitioning(workersNum, labels);
+		return new Result(pfs, pls);
+	}
+}

http://git-wip-us.apache.org/repos/asf/systemml/blob/8def2040/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DRRScheme.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DRRScheme.java b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DRRScheme.java
new file mode 100644
index 0000000..90c62d6
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DRRScheme.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.controlprogram.paramserv;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.LongStream;
+
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.util.DataConverter;
+
+/**
+ * Disjoint_Round_Robin data partitioner:
+ * for each worker, use a permutation multiply
+ * or simpler a removeEmpty such as removeEmpty
+ * (target=X, margin=rows, select=(seq(1,nrow(X))%%k)==id)
+ */
+public class DRRScheme extends DataPartitionScheme {
+
+	public static MatrixBlock removeEmpty(MatrixBlock mb, int k, int workerId) {
+		double[] data = LongStream.range(0, mb.getNumRows()).mapToDouble(l -> l % k == workerId ? 1 : 0).toArray();
+		MatrixBlock select = DataConverter.convertToMatrixBlock(data, true);
+		return mb.removeEmptyOperations(new MatrixBlock(), true, true, select);
+	}
+
+	private MatrixObject internalRemoveEmpty(MatrixBlock mb, int k, int workerId) {
+		MatrixObject result = ParamservUtils.newMatrixObject(removeEmpty(mb, k, workerId));
+		result.enableCleanup(false);
+		return result;
+	}
+
+	@Override
+	public Result doPartitioning(int workersNum, MatrixBlock features, MatrixBlock labels) {
+		List<MatrixObject> pfs = IntStream.range(0, workersNum).mapToObj(i -> internalRemoveEmpty(features, workersNum, i)).collect(Collectors.toList());
+		List<MatrixObject> pls = IntStream.range(0, workersNum).mapToObj(i -> internalRemoveEmpty(labels, workersNum, i)).collect(Collectors.toList());
+		return new Result(pfs, pls);
+	}
+}

http://git-wip-us.apache.org/repos/asf/systemml/blob/8def2040/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DRScheme.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DRScheme.java b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DRScheme.java
new file mode 100644
index 0000000..062a7ab
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DRScheme.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.controlprogram.paramserv;
+
+import static org.apache.sysml.runtime.controlprogram.paramserv.ParamservUtils.SEED;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+
+/**
+ * Data partitioner Disjoint_Random:
+ * for each worker, use a permutation multiply P[beg:end,] %*% X,
+ * where P is constructed for example with P=table(seq(1,nrow(X)),sample(nrow(X), nrow(X))),
+ * i.e., sampling without replacement to ensure disjointness.
+ */
+public class DRScheme extends DataPartitionScheme {
+
+	private List<MatrixBlock> partition(int k, MatrixBlock mb, MatrixBlock permutation) {
+		int batchSize = (int) Math.ceil((double) mb.getNumRows() / k);
+		return IntStream.range(0, k).mapToObj(i -> {
+			int begin = i * batchSize;
+			int end = Math.min((i + 1) * batchSize, mb.getNumRows());
+			MatrixBlock slicedPerm = permutation.slice(begin, end - 1);
+			return slicedPerm.aggregateBinaryOperations(slicedPerm, mb, new MatrixBlock(), InstructionUtils.getMatMultOperator(k));
+		}).collect(Collectors.toList());
+	}
+
+	private List<MatrixObject> internalDoPartitioning(int k, MatrixBlock mb, MatrixBlock permutation) {
+		return partition(k, mb, permutation).stream().map(ParamservUtils::newMatrixObject).collect(Collectors.toList());
+	}
+
+	@Override
+	public Result doPartitioning(int workersNum, MatrixBlock features, MatrixBlock labels) {
+		// Generate a single permutation matrix (workers use slices)
+		MatrixBlock permutation = ParamservUtils.generatePermutation(features.getNumRows(), SEED);
+		List<MatrixObject> pfs = internalDoPartitioning(workersNum, features, permutation);
+		List<MatrixObject> pls = internalDoPartitioning(workersNum, labels, permutation);
+		return new Result(pfs, pls);
+	}
+}

http://git-wip-us.apache.org/repos/asf/systemml/blob/8def2040/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DataPartitionScheme.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DataPartitionScheme.java b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DataPartitionScheme.java
new file mode 100644
index 0000000..f2ea0aa
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DataPartitionScheme.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.controlprogram.paramserv;
+
+import java.util.List;
+
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+
+public abstract class DataPartitionScheme {
+
+	public final class Result {
+		public final List<MatrixObject> pFeatures;
+		public final List<MatrixObject> pLabels;
+
+		public Result(List<MatrixObject> pFeatures, List<MatrixObject> pLabels) {
+			this.pFeatures = pFeatures;
+			this.pLabels = pLabels;
+		}
+	}
+
+	public abstract Result doPartitioning(int workersNum, MatrixBlock features, MatrixBlock labels);
+}

http://git-wip-us.apache.org/repos/asf/systemml/blob/8def2040/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DataPartitioner.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DataPartitioner.java b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DataPartitioner.java
index fce6d35..3f28cd1 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DataPartitioner.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DataPartitioner.java
@@ -19,22 +19,31 @@
 
 package org.apache.sysml.runtime.controlprogram.paramserv;
 
-import java.util.List;
-
-import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
-
-public abstract class DataPartitioner {
-
-	public final class Result {
-		public final List<MatrixObject> pFeatures;
-		public final List<MatrixObject> pLabels;
-
-		public Result(List<MatrixObject> pFeatures, List<MatrixObject> pLabels) {
-			this.pFeatures = pFeatures;
-			this.pLabels = pLabels;
+import org.apache.sysml.parser.Statement;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+
+public class DataPartitioner {
+
+	private DataPartitionScheme _scheme;
+
+	public DataPartitioner(Statement.PSScheme scheme) {
+		switch (scheme) {
+			case DISJOINT_CONTIGUOUS:
+				_scheme = new DCScheme();
+				break;
+			case DISJOINT_ROUND_ROBIN:
+				_scheme = new DRRScheme();
+				break;
+			case DISJOINT_RANDOM:
+				_scheme = new DRScheme();
+				break;
+			case OVERLAP_RESHUFFLE:
+				_scheme = new ORScheme();
+				break;
 		}
 	}
 
-	public abstract Result doPartitioning(int workersNum, MatrixObject features, MatrixObject labels);
-
+	public DataPartitionScheme.Result doPartitioning(int workersNum, MatrixBlock features, MatrixBlock labels) {
+		return _scheme.doPartitioning(workersNum, features, labels);
+	}
 }

http://git-wip-us.apache.org/repos/asf/systemml/blob/8def2040/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DataPartitionerDC.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DataPartitionerDC.java b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DataPartitionerDC.java
deleted file mode 100644
index 4810541..0000000
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DataPartitionerDC.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.runtime.controlprogram.paramserv;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
-
-/**
- * Disjoint_Contiguous data partitioner:
- *
- * for each worker, use a right indexing
- * operation X[beg:end,] to obtain contiguous,
- * non-overlapping partitions of rows.
- */
-public class DataPartitionerDC extends DataPartitioner {
-
-	private List<MatrixObject> doPartitioning(int k, MatrixObject mo) {
-		List<MatrixObject> list = new ArrayList<>();
-		long stepSize = (long) Math.ceil((double) mo.getNumRows() / k);
-		long begin = 1;
-		while (begin < mo.getNumRows()) {
-			long end = Math.min(begin - 1 + stepSize, mo.getNumRows());
-			MatrixObject pmo = ParamservUtils.sliceMatrix(mo, begin, end);
-			list.add(pmo);
-			begin = end + 1;
-		}
-		return list;
-	}
-
-	@Override
-	public Result doPartitioning(int workersNum, MatrixObject features, MatrixObject labels) {
-		List<MatrixObject> pfs = doPartitioning(workersNum, features);
-		List<MatrixObject> pls = doPartitioning(workersNum, labels);
-		return new Result(pfs, pls);
-	}
-}

http://git-wip-us.apache.org/repos/asf/systemml/blob/8def2040/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DataPartitionerDR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DataPartitionerDR.java b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DataPartitionerDR.java
deleted file mode 100644
index adc6f60..0000000
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DataPartitionerDR.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.runtime.controlprogram.paramserv;
-
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-
-import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
-import org.apache.sysml.runtime.instructions.InstructionUtils;
-import org.apache.sysml.runtime.matrix.data.MatrixBlock;
-
-/**
- * Data partitioner Disjoint_Random:
- * for each worker, use a permutation multiply P[beg:end,] %*% X,
- * where P is constructed for example with P=table(seq(1,nrow(X)),sample(nrow(X), nrow(X))),
- * i.e., sampling without replacement to ensure disjointness.
- */
-public class DataPartitionerDR extends DataPartitioner {
-
-	private List<MatrixObject> doPartitioning(int k, MatrixObject mo, MatrixBlock permutation) {
-		MatrixBlock data = mo.acquireRead();
-		int batchSize = (int) Math.ceil((double) mo.getNumRows() / k);
-		List<MatrixObject> pMatrices = IntStream.range(0, k).mapToObj(i -> {
-			int begin = i * batchSize;
-			int end = (int) Math.min((i + 1) * batchSize, mo.getNumRows());
-			MatrixBlock slicedPerm = permutation.slice(begin, end - 1);
-			MatrixBlock output = slicedPerm.aggregateBinaryOperations(slicedPerm,
-				data, new MatrixBlock(), InstructionUtils.getMatMultOperator(k));
-			MatrixObject result = ParamservUtils.newMatrixObject();
-			result.acquireModify(output);
-			result.release();
-			return result;
-		}).collect(Collectors.toList());
-		mo.release();
-		return pMatrices;
-	}
-
-	@Override
-	public Result doPartitioning(int workersNum, MatrixObject features, MatrixObject labels) {
-		// Generate a single permutation matrix (workers use slices)
-		MatrixBlock permutation = ParamservUtils.generatePermutation((int)features.getNumRows());
-		List<MatrixObject> pfs = doPartitioning(workersNum, features, permutation);
-		List<MatrixObject> pls = doPartitioning(workersNum, labels, permutation);
-		return new Result(pfs, pls);
-	}
-}

http://git-wip-us.apache.org/repos/asf/systemml/blob/8def2040/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DataPartitionerDRR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DataPartitionerDRR.java b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DataPartitionerDRR.java
deleted file mode 100644
index a2ff5f9..0000000
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DataPartitionerDRR.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.runtime.controlprogram.paramserv;
-
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-import java.util.stream.LongStream;
-
-import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
-import org.apache.sysml.runtime.matrix.data.MatrixBlock;
-import org.apache.sysml.runtime.util.DataConverter;
-
-/**
- * Disjoint_Round_Robin data partitioner:
- * for each worker, use a permutation multiply
- * or simpler a removeEmpty such as removeEmpty
- * (target=X, margin=rows, select=(seq(1,nrow(X))%%k)==id)
- */
-public class DataPartitionerDRR extends DataPartitioner {
-
-	private MatrixObject removeEmpty(MatrixObject mo, int k, int workerId) {
-		MatrixObject result = ParamservUtils.newMatrixObject();
-		MatrixBlock tmp = mo.acquireRead();
-		double[] data = LongStream.range(1, mo.getNumRows() + 1)
-			.mapToDouble(l -> l % k == workerId ? 1 : 0).toArray();
-		MatrixBlock select = DataConverter.convertToMatrixBlock(data, true);
-		MatrixBlock resultMB = tmp.removeEmptyOperations(new MatrixBlock(), true, true, select);
-		mo.release();
-		result.acquireModify(resultMB);
-		result.release();
-		result.enableCleanup(false);
-		return result;
-	}
-
-	@Override
-	public Result doPartitioning(int workersNum, MatrixObject features, MatrixObject labels) {
-		List<MatrixObject> pfs = IntStream.range(0, workersNum)
-	  		.mapToObj(i -> removeEmpty(features, workersNum, i)).collect(Collectors.toList());
-		List<MatrixObject> pls = IntStream.range(0, workersNum)
-		    .mapToObj(i -> removeEmpty(labels, workersNum, i)).collect(Collectors.toList());
-		return new Result(pfs, pls);
-	}
-}

http://git-wip-us.apache.org/repos/asf/systemml/blob/8def2040/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DataPartitionerOR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DataPartitionerOR.java b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DataPartitionerOR.java
deleted file mode 100644
index 0bfb4b7..0000000
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/DataPartitionerOR.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.runtime.controlprogram.paramserv;
-
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-
-import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
-import org.apache.sysml.runtime.instructions.InstructionUtils;
-import org.apache.sysml.runtime.matrix.data.MatrixBlock;
-
-/**
- * Data partitioner Overlap_Reshuffle:
- * for each worker, use a new permutation multiply P %*% X,
- * where P is constructed for example with P=table(seq(1,nrow(X),sample(nrow(X), nrow(X))))
- */
-public class DataPartitionerOR extends DataPartitioner {
-
-	private List<MatrixObject> doPartitioning(int k, MatrixObject mo, List<MatrixBlock> permutations) {
-		MatrixBlock data = mo.acquireRead();
-		List<MatrixObject> pMatrices = IntStream.range(0, k).mapToObj(i -> {
-			MatrixBlock permutation = permutations.get(i);
-			MatrixBlock output = permutation.aggregateBinaryOperations(permutation,
-				data, new MatrixBlock(), InstructionUtils.getMatMultOperator(k));
-			MatrixObject result = ParamservUtils.newMatrixObject();
-			result.acquireModify(output);
-			result.release();
-			return result;
-		}).collect(Collectors.toList());
-		mo.release();
-		return pMatrices;
-	}
-
-	@Override
-	public Result doPartitioning(int workersNum, MatrixObject features, MatrixObject labels) {
-		// Generate a different permutation matrix for each worker
-		List<MatrixBlock> permutations = IntStream.range(0, workersNum)
-			.mapToObj(i -> ParamservUtils.generatePermutation((int)features.getNumRows()))
-	    	.collect(Collectors.toList());
-		List<MatrixObject> pfs = doPartitioning(workersNum, features, permutations);
-		List<MatrixObject> pls = doPartitioning(workersNum, labels, permutations);
-		return new Result(pfs, pls);
-	}
-}

http://git-wip-us.apache.org/repos/asf/systemml/blob/8def2040/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/LocalPSWorker.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/LocalPSWorker.java b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/LocalPSWorker.java
index 366284c..307669e 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/LocalPSWorker.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/LocalPSWorker.java
@@ -187,5 +187,4 @@ public class LocalPSWorker extends PSWorker implements Callable<Void> {
 		ParamservUtils.cleanupData(bLabels);
 		return gradients;
 	}
-
 }

http://git-wip-us.apache.org/repos/asf/systemml/blob/8def2040/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ORScheme.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ORScheme.java b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ORScheme.java
new file mode 100644
index 0000000..2692efa
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ORScheme.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.controlprogram.paramserv;
+
+import static org.apache.sysml.runtime.controlprogram.paramserv.ParamservUtils.SEED;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+
+/**
+ * Data partitioner Overlap_Reshuffle:
+ * for each worker, use a new permutation multiply P %*% X,
+ * where P is constructed for example with P=table(seq(1,nrow(X),sample(nrow(X), nrow(X))))
+ */
+public class ORScheme extends DataPartitionScheme {
+
+	public static List<MatrixBlock> partition(int k, MatrixBlock mb, List<MatrixBlock> permutations) {
+		return IntStream.range(0, k).mapToObj(i -> {
+			MatrixBlock permutation = permutations.get(i);
+			return permutation.aggregateBinaryOperations(permutation, mb, new MatrixBlock(),
+					InstructionUtils.getMatMultOperator(k));
+		}).collect(Collectors.toList());
+	}
+
+	private List<MatrixObject> doPartitioning(int k, MatrixBlock mb, List<MatrixBlock> permutations) {
+		return partition(k, mb, permutations).stream().map(ParamservUtils::newMatrixObject).collect(Collectors.toList());
+	}
+
+	@Override
+	public Result doPartitioning(int workersNum, MatrixBlock features, MatrixBlock labels) {
+		// Generate a different permutation matrix for each worker
+		List<MatrixBlock> permutations = IntStream.range(0, workersNum)
+			.mapToObj(i -> ParamservUtils.generatePermutation(features.getNumRows(), SEED+i))
+			.collect(Collectors.toList());
+		List<MatrixObject> pfs = doPartitioning(workersNum, features, permutations);
+		List<MatrixObject> pls = doPartitioning(workersNum, labels, permutations);
+		return new Result(pfs, pls);
+	}
+}

http://git-wip-us.apache.org/repos/asf/systemml/blob/8def2040/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ParamservUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ParamservUtils.java b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ParamservUtils.java
index 3aee170..ec6b6b2 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ParamservUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ParamservUtils.java
@@ -22,11 +22,15 @@ package org.apache.sysml.runtime.controlprogram.paramserv;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashSet;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
 import org.apache.commons.lang.StringUtils;
+import org.apache.spark.Partitioner;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.sysml.conf.ConfigurationManager;
 import org.apache.sysml.hops.Hop;
 import org.apache.sysml.hops.MultiThreadedHop;
 import org.apache.sysml.hops.OptimizerUtils;
@@ -34,6 +38,7 @@ import org.apache.sysml.hops.recompile.Recompiler;
 import org.apache.sysml.parser.DMLProgram;
 import org.apache.sysml.parser.DMLTranslator;
 import org.apache.sysml.parser.Expression;
+import org.apache.sysml.parser.Statement;
 import org.apache.sysml.parser.StatementBlock;
 import org.apache.sysml.runtime.DMLRuntimeException;
 import org.apache.sysml.runtime.controlprogram.ForProgramBlock;
@@ -49,6 +54,9 @@ import org.apache.sysml.runtime.controlprogram.caching.FrameObject;
 import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
 import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
 import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.controlprogram.paramserv.spark.DataPartitionerSparkAggregator;
+import org.apache.sysml.runtime.controlprogram.paramserv.spark.DataPartitionerSparkMapper;
 import org.apache.sysml.runtime.controlprogram.parfor.ProgramConverter;
 import org.apache.sysml.runtime.functionobjects.Plus;
 import org.apache.sysml.runtime.instructions.cp.Data;
@@ -57,13 +65,18 @@ import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
 import org.apache.sysml.runtime.matrix.MetaDataFormat;
 import org.apache.sysml.runtime.matrix.data.InputInfo;
 import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
 import org.apache.sysml.runtime.matrix.data.OutputInfo;
 import org.apache.sysml.runtime.matrix.operators.BinaryOperator;
 
+import scala.Tuple2;
+
 public class ParamservUtils {
 
 	public static final String PS_FUNC_PREFIX = "_ps_";
 
+	public static long SEED = -1; // Used for generating permutation
+
 	/**
 	 * Deep copy the list object
 	 *
@@ -105,8 +118,13 @@ public class ParamservUtils {
 		cd.clearData();
 	}
 
-	public static MatrixObject newMatrixObject() {
-		return new MatrixObject(Expression.ValueType.DOUBLE, OptimizerUtils.getUniqueTempFileName(), new MetaDataFormat(new MatrixCharacteristics(-1, -1, -1, -1), OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo));
+	public static MatrixObject newMatrixObject(MatrixBlock mb) {
+		MatrixObject result = new MatrixObject(Expression.ValueType.DOUBLE, OptimizerUtils.getUniqueTempFileName(),
+			new MetaDataFormat(new MatrixCharacteristics(-1, -1, ConfigurationManager.getBlocksize(),
+			ConfigurationManager.getBlocksize()), OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo));
+		result.acquireModify(mb);
+		result.release();
+		return result;
 	}
 
 	/**
@@ -118,20 +136,31 @@ public class ParamservUtils {
 	 * @return new sliced matrix
 	 */
 	public static MatrixObject sliceMatrix(MatrixObject mo, long rl, long rh) {
-		MatrixObject result = newMatrixObject();
-		MatrixBlock tmp = mo.acquireRead();
-		result.acquireModify(tmp.slice((int) rl - 1, (int) rh - 1));
-		mo.release();
-		result.release();
+		MatrixBlock mb = mo.acquireRead();
+		MatrixObject result = newMatrixObject(sliceMatrixBlock(mb, rl, rh));
 		result.enableCleanup(false);
+		mo.release();
 		return result;
 	}
 
-	public static MatrixBlock generatePermutation(int numEntries) {
+	/**
+	 * Slice the matrix block and return a matrix block
+	 * (used in spark)
+	 *
+	 * @param mb input matrix
+	 * @param rl low boundary
+	 * @param rh high boundary
+	 * @return new sliced matrix block
+	 */
+	public static MatrixBlock sliceMatrixBlock(MatrixBlock mb, long rl, long rh) {
+		return mb.slice((int) rl - 1, (int) rh - 1);
+	}
+
+	public static MatrixBlock generatePermutation(int numEntries, long seed) {
 		// Create a sequence and sample w/o replacement
 		// (no need to materialize the sequence because ctable only uses its meta data)
 		MatrixBlock seq = new MatrixBlock(numEntries, 1, false);
-		MatrixBlock sample = MatrixBlock.sampleOperations(numEntries, numEntries, false, -1);
+		MatrixBlock sample = MatrixBlock.sampleOperations(numEntries, numEntries, false, seed);
 
 		// Combine the sequence and sample as a table
 		return seq.ctableSeqOperations(sample, 1.0,
@@ -257,14 +286,104 @@ public class ParamservUtils {
 		return recompiled;
 	}
 
-
 	private static FunctionProgramBlock getFunctionBlock(ExecutionContext ec, String funcName) {
 		String[] cfn = getCompleteFuncName(funcName, null);
 		String ns = cfn[0];
 		String fname = cfn[1];
 		return ec.getProgram().getFunctionProgramBlock(ns, fname);
 	}
-	
+
+	public static MatrixBlock cbindMatrix(MatrixBlock left, MatrixBlock right) {
+		return left.append(right, new MatrixBlock());
+	}
+
+	/**
+	 * Assemble the matrix of features and labels according to the rowID
+	 *
+	 * @param numRows row size of the data
+	 * @param featuresRDD indexed features matrix block
+	 * @param labelsRDD indexed labels matrix block
+	 * @return Assembled rdd with rowID as key while matrix of features and labels as value (rowID -> features, labels)
+	 */
+	public static JavaPairRDD<Long, Tuple2<MatrixBlock, MatrixBlock>> assembleTrainingData(long numRows, JavaPairRDD<MatrixIndexes, MatrixBlock> featuresRDD, JavaPairRDD<MatrixIndexes, MatrixBlock> labelsRDD) {
+		JavaPairRDD<Long, MatrixBlock> fRDD = groupMatrix(numRows, featuresRDD);
+		JavaPairRDD<Long, MatrixBlock> lRDD = groupMatrix(numRows, labelsRDD);
+		//TODO Add an additional physical operator which broadcasts the labels directly (broadcast join with features) if certain memory budgets are satisfied
+		return fRDD.join(lRDD);
+	}
+
+	private static JavaPairRDD<Long, MatrixBlock> groupMatrix(long numRows, JavaPairRDD<MatrixIndexes, MatrixBlock> rdd) {
+		//TODO could use join and aggregation to avoid unnecessary shuffle introduced by reduceByKey
+		return rdd.mapToPair(input -> new Tuple2<>(input._1.getRowIndex(), new Tuple2<>(input._1.getColumnIndex(), input._2)))
+			.aggregateByKey(new LinkedList<Tuple2<Long, MatrixBlock>>(),
+				new Partitioner() {
+					private static final long serialVersionUID = -7032660778344579236L;
+					@Override
+					public int getPartition(Object rblkID) {
+						return Math.toIntExact((Long) rblkID);
+					}
+					@Override
+					public int numPartitions() {
+						return Math.toIntExact(numRows);
+					}
+				},
+				(list, input) -> {
+					list.add(input); 
+					return list;
+				}, 
+				(l1, l2) -> {
+					l1.addAll(l2);
+					l1.sort((o1, o2) -> o1._1.compareTo(o2._1));
+					return l1;
+				})
+			.mapToPair(input -> {
+				LinkedList<Tuple2<Long, MatrixBlock>> list = input._2;
+				MatrixBlock result = list.get(0)._2;
+				for (int i = 1; i < list.size(); i++) {
+					result = ParamservUtils.cbindMatrix(result, list.get(i)._2);
+				}
+				return new Tuple2<>(input._1, result);
+			});
+	}
+
+	@SuppressWarnings("unchecked")
+	public static JavaPairRDD<Integer, Tuple2<MatrixBlock, MatrixBlock>> doPartitionOnSpark(SparkExecutionContext sec, MatrixObject features, MatrixObject labels, Statement.PSScheme scheme, int workerNum) {
+		// Get input RDD
+		JavaPairRDD<MatrixIndexes, MatrixBlock> featuresRDD = (JavaPairRDD<MatrixIndexes, MatrixBlock>)
+				sec.getRDDHandleForMatrixObject(features, InputInfo.BinaryBlockInputInfo);
+		JavaPairRDD<MatrixIndexes, MatrixBlock> labelsRDD = (JavaPairRDD<MatrixIndexes, MatrixBlock>)
+				sec.getRDDHandleForMatrixObject(labels, InputInfo.BinaryBlockInputInfo);
+
+		DataPartitionerSparkMapper mapper = new DataPartitionerSparkMapper(scheme, workerNum, sec, (int) features.getNumRows());
+		return ParamservUtils.assembleTrainingData(features.getNumRows(), featuresRDD, labelsRDD) // Combine features and labels into a pair (rowBlockID => (features, labels))
+			.flatMapToPair(mapper) // Do the data partitioning on spark (workerID => (rowBlockID, (single row features, single row labels))
+			// Aggregate the partitioned matrix according to rowID for each worker
+			// i.e. (workerID => ordered list[(rowBlockID, (single row features, single row labels)]
+			.aggregateByKey(new LinkedList<Tuple2<Long, Tuple2<MatrixBlock, MatrixBlock>>>(),
+				new Partitioner() {
+					private static final long serialVersionUID = -7937781374718031224L;
+					@Override
+					public int getPartition(Object workerID) {
+						return (int) workerID;
+					}
+					@Override
+					public int numPartitions() {
+						return workerNum;
+					}
+				}, 
+				(list, input) -> {
+					list.add(input);
+					return list;
+				},
+				(l1, l2) -> {
+					l1.addAll(l2);
+					l1.sort((o1, o2) -> o1._1.compareTo(o2._1));
+					return l1;
+				})
+			.mapToPair(new DataPartitionerSparkAggregator(
+				features.getNumColumns(), labels.getNumColumns()));
+	}
+
 	public static ListObject accrueGradients(ListObject accGradients, ListObject gradients) {
 		return accrueGradients(accGradients, gradients, false);
 	}

http://git-wip-us.apache.org/repos/asf/systemml/blob/8def2040/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/DCSparkScheme.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/DCSparkScheme.java b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/DCSparkScheme.java
new file mode 100644
index 0000000..666b891
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/DCSparkScheme.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.controlprogram.paramserv.spark;
+
+import java.util.List;
+
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+
+import scala.Tuple2;
+
+/**
+ * Spark Disjoint_Contiguous data partitioner:
+ * <p>
+ * For each row, find out the shifted place according to the workerID indicator
+ */
+public class DCSparkScheme extends DataPartitionSparkScheme {
+
+	private static final long serialVersionUID = -2786906947020788787L;
+
+	protected DCSparkScheme() {
+		// No-args constructor used for deserialization
+	}
+
+	@Override
+	public Result doPartitioning(int numWorkers, int rblkID, MatrixBlock features, MatrixBlock labels) {
+		List<Tuple2<Integer, Tuple2<Long, MatrixBlock>>> pfs = nonShuffledPartition(rblkID, features);
+		List<Tuple2<Integer, Tuple2<Long, MatrixBlock>>> pls = nonShuffledPartition(rblkID, labels);
+		return new Result(pfs, pls);
+	}
+}

http://git-wip-us.apache.org/repos/asf/systemml/blob/8def2040/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/DRRSparkScheme.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/DRRSparkScheme.java b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/DRRSparkScheme.java
new file mode 100644
index 0000000..7683251
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/DRRSparkScheme.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.controlprogram.paramserv.spark;
+
+import java.util.List;
+
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+
+import scala.Tuple2;
+
+/**
+ * Spark Disjoint_Round_Robin data partitioner:
+ */
+public class DRRSparkScheme extends DataPartitionSparkScheme {
+
+	private static final long serialVersionUID = -3130831851505549672L;
+
+	protected DRRSparkScheme() {
+		// No-args constructor used for deserialization
+	}
+
+	@Override
+	public Result doPartitioning(int numWorkers, int rblkID, MatrixBlock features, MatrixBlock labels) {
+		List<Tuple2<Integer, Tuple2<Long, MatrixBlock>>> pfs = nonShuffledPartition(rblkID, features);
+		List<Tuple2<Integer, Tuple2<Long, MatrixBlock>>> pls = nonShuffledPartition(rblkID, labels);
+		return new Result(pfs, pls);
+	}
+}

http://git-wip-us.apache.org/repos/asf/systemml/blob/8def2040/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/DRSparkScheme.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/DRSparkScheme.java b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/DRSparkScheme.java
new file mode 100644
index 0000000..51cc523
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/DRSparkScheme.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.controlprogram.paramserv.spark;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.controlprogram.paramserv.ParamservUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+
+import scala.Tuple2;
+
+/**
+ * Spark data partitioner Disjoint_Random:
+ *
+ * For the current row block, find all the shifted place for each row (WorkerID => (row block ID, matrix)
+ */
+public class DRSparkScheme extends DataPartitionSparkScheme {
+
+	private static final long serialVersionUID = -7655310624144544544L;
+
+	protected DRSparkScheme() {
+		// No-args constructor used for deserialization
+	}
+
+	@Override
+	public Result doPartitioning(int numWorkers, int rblkID, MatrixBlock features, MatrixBlock labels) {
+		List<Tuple2<Integer, Tuple2<Long, MatrixBlock>>> pfs = partition(rblkID, features);
+		List<Tuple2<Integer, Tuple2<Long, MatrixBlock>>> pls = partition(rblkID, labels);
+		return new Result(pfs, pls);
+	}
+
+	private List<Tuple2<Integer, Tuple2<Long, MatrixBlock>>> partition(int rblkID, MatrixBlock mb) {
+		MatrixBlock partialPerm = _globalPerms.get(0).getBlock(rblkID, 1);
+
+		// For each row, find out the shifted place
+		return IntStream.range(0, mb.getNumRows()).mapToObj(r -> {
+			MatrixBlock rowMB = ParamservUtils.sliceMatrixBlock(mb, r + 1, r + 1);
+			long shiftedPosition = (long) partialPerm.getValue(r, 0);
+
+			// Get the shifted block and position
+			int shiftedBlkID = (int) (shiftedPosition / OptimizerUtils.DEFAULT_BLOCKSIZE + 1);
+
+			MatrixBlock indicator = _workerIndicator.getBlock(shiftedBlkID, 1);
+			int workerID = (int) indicator.getValue((int) shiftedPosition / OptimizerUtils.DEFAULT_BLOCKSIZE, 0);
+			return new Tuple2<>(workerID, new Tuple2<>(shiftedPosition, rowMB));
+		}).collect(Collectors.toList());
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/systemml/blob/8def2040/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/DataPartitionSparkScheme.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/DataPartitionSparkScheme.java b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/DataPartitionSparkScheme.java
new file mode 100644
index 0000000..9875dd2
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/DataPartitionSparkScheme.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.controlprogram.paramserv.spark;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.LongStream;
+
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.controlprogram.paramserv.ParamservUtils;
+import org.apache.sysml.runtime.instructions.spark.data.PartitionedBroadcast;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+
+import scala.Tuple2;
+
+public abstract class DataPartitionSparkScheme implements Serializable {
+
+	protected final class Result {
+		protected final List<Tuple2<Integer, Tuple2<Long, MatrixBlock>>> pFeatures; // WorkerID => (rowID, matrix)
+		protected final List<Tuple2<Integer, Tuple2<Long, MatrixBlock>>> pLabels;
+
+		protected Result(List<Tuple2<Integer, Tuple2<Long, MatrixBlock>>> pFeatures, List<Tuple2<Integer, Tuple2<Long, MatrixBlock>>> pLabels) {
+			this.pFeatures = pFeatures;
+			this.pLabels = pLabels;
+		}
+	}
+
+	private static final long serialVersionUID = -3462829818083371171L;
+
+	protected List<PartitionedBroadcast<MatrixBlock>> _globalPerms; // a list of global permutations
+	protected PartitionedBroadcast<MatrixBlock> _workerIndicator; // a matrix indicating to which worker the given row belongs
+
+	protected void setGlobalPermutation(List<PartitionedBroadcast<MatrixBlock>> gps) {
+		_globalPerms = gps;
+	}
+
+	protected void setWorkerIndicator(PartitionedBroadcast<MatrixBlock> wi) {
+		_workerIndicator = wi;
+	}
+
+	/**
+	 * Do non-reshuffled data partitioning according to worker indicator
+	 * @param rblkID row block ID
+	 * @param mb Matrix
+	 * @return list of tuple (workerID, (row block ID, matrix row))
+	 */
+	protected List<Tuple2<Integer, Tuple2<Long, MatrixBlock>>> nonShuffledPartition(int rblkID, MatrixBlock mb) {
+		MatrixBlock indicator = _workerIndicator.getBlock(rblkID, 1);
+		return LongStream.range(0, mb.getNumRows()).mapToObj(r -> {
+			int workerID = (int) indicator.getValue((int) r, 0);
+			MatrixBlock rowMB = ParamservUtils.sliceMatrixBlock(mb, r + 1, r + 1);
+			long shiftedPosition = r + (rblkID - 1) * OptimizerUtils.DEFAULT_BLOCKSIZE;
+			return new Tuple2<>(workerID, new Tuple2<>(shiftedPosition, rowMB));
+		}).collect(Collectors.toList());
+	}
+
+	public abstract Result doPartitioning(int numWorkers, int rblkID, MatrixBlock features, MatrixBlock labels);
+}

http://git-wip-us.apache.org/repos/asf/systemml/blob/8def2040/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/DataPartitionerSparkAggregator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/DataPartitionerSparkAggregator.java b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/DataPartitionerSparkAggregator.java
new file mode 100644
index 0000000..39b8adf
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/DataPartitionerSparkAggregator.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.controlprogram.paramserv.spark;
+
+import java.io.Serializable;
+import java.util.LinkedList;
+
+import org.apache.spark.api.java.function.PairFunction;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+
+import scala.Tuple2;
+
+public class DataPartitionerSparkAggregator implements PairFunction<Tuple2<Integer,LinkedList<Tuple2<Long,Tuple2<MatrixBlock,MatrixBlock>>>>, Integer, Tuple2<MatrixBlock, MatrixBlock>>, Serializable {
+
+	private static final long serialVersionUID = -1245300852709085117L;
+	private long _fcol;
+	private long _lcol;
+
+	public DataPartitionerSparkAggregator() {
+
+	}
+
+	public DataPartitionerSparkAggregator(long fcol, long lcol) {
+		_fcol = fcol;
+		_lcol = lcol;
+	}
+
+	/**
+	 * Row-wise combine the matrix
+	 * @param input workerID => ordered list [(rowBlockID, (features, labels))]
+	 * @return workerID => [(features, labels)]
+	 * @throws Exception Some exception
+	 */
+	@Override
+	public Tuple2<Integer, Tuple2<MatrixBlock, MatrixBlock>> call(Tuple2<Integer, LinkedList<Tuple2<Long, Tuple2<MatrixBlock, MatrixBlock>>>> input) throws Exception {
+		MatrixBlock fmb = new MatrixBlock(input._2.size(), (int) _fcol, false);
+		MatrixBlock lmb = new MatrixBlock(input._2.size(), (int) _lcol, false);
+
+		for (int i = 0; i < input._2.size(); i++) {
+			MatrixBlock tmpFMB = input._2.get(i)._2._1;
+			MatrixBlock tmpLMB = input._2.get(i)._2._2;
+			// Row-wise aggregation
+			fmb = fmb.leftIndexingOperations(tmpFMB, i, i, 0, (int) _fcol - 1, fmb, MatrixObject.UpdateType.INPLACE_PINNED);
+			lmb = lmb.leftIndexingOperations(tmpLMB, i, i, 0, (int) _lcol - 1, lmb, MatrixObject.UpdateType.INPLACE_PINNED);
+		}
+		return new Tuple2<>(input._1, new Tuple2<>(fmb, lmb));
+	}
+}

http://git-wip-us.apache.org/repos/asf/systemml/blob/8def2040/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/DataPartitionerSparkMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/DataPartitionerSparkMapper.java b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/DataPartitionerSparkMapper.java
new file mode 100644
index 0000000..2a69986
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/DataPartitionerSparkMapper.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.controlprogram.paramserv.spark;
+
+import java.io.Serializable;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.spark.api.java.function.PairFlatMapFunction;
+import org.apache.sysml.parser.Statement;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+
+import scala.Tuple2;
+
+public class DataPartitionerSparkMapper implements PairFlatMapFunction<Tuple2<Long, Tuple2<MatrixBlock, MatrixBlock>>, Integer, Tuple2<Long, Tuple2<MatrixBlock, MatrixBlock>>>, Serializable {
+
+	private static final long serialVersionUID = 1710721606050403296L;
+	private int _workersNum;
+
+	private SparkDataPartitioner _dp;
+
+	protected DataPartitionerSparkMapper() {
+		// No-args constructor used for deserialization
+	}
+
+	public DataPartitionerSparkMapper(Statement.PSScheme scheme, int workersNum, SparkExecutionContext sec, int numEntries) {
+		_workersNum = workersNum;
+		_dp = new SparkDataPartitioner(scheme, sec, numEntries, workersNum);
+	}
+
+	/**
+	 * Do data partitioning
+	 * @param input RowBlockID => (features, labels)
+	 * @return WorkerID => (rowBlockID, (single row features, single row labels))
+	 * @throws Exception Some exception
+	 */
+	@Override
+	public Iterator<Tuple2<Integer, Tuple2<Long, Tuple2<MatrixBlock, MatrixBlock>>>> call(Tuple2<Long,Tuple2<MatrixBlock,MatrixBlock>> input)
+			throws Exception {
+		List<Tuple2<Integer, Tuple2<Long,Tuple2<MatrixBlock,MatrixBlock>>>> partitions = new LinkedList<>();
+		MatrixBlock features = input._2._1;
+		MatrixBlock labels = input._2._2;
+		DataPartitionSparkScheme.Result result = _dp.doPartitioning(_workersNum, features, labels, input._1);
+		for (int i = 0; i < result.pFeatures.size(); i++) {
+			Tuple2<Integer, Tuple2<Long, MatrixBlock>> ft = result.pFeatures.get(i);
+			Tuple2<Integer, Tuple2<Long, MatrixBlock>> lt = result.pLabels.get(i);
+			partitions.add(new Tuple2<>(ft._1, new Tuple2<>(ft._2._1, new Tuple2<>(ft._2._2, lt._2._2))));
+		}
+		return partitions.iterator();
+	}
+}

http://git-wip-us.apache.org/repos/asf/systemml/blob/8def2040/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/ORSparkScheme.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/ORSparkScheme.java b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/ORSparkScheme.java
new file mode 100644
index 0000000..16ce516
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/ORSparkScheme.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.controlprogram.paramserv.spark;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.apache.sysml.runtime.controlprogram.paramserv.ParamservUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+
+import scala.Tuple2;
+
+/**
+ * Spark data partitioner Overlap_Reshuffle:
+ *
+ */
+public class ORSparkScheme extends DataPartitionSparkScheme {
+
+	private static final long serialVersionUID = 6867567406403580311L;
+
+	protected ORSparkScheme() {
+		// No-args constructor used for deserialization
+	}
+
+	@Override
+	public Result doPartitioning(int numWorkers, int rblkID, MatrixBlock features, MatrixBlock labels) {
+		List<Tuple2<Integer, Tuple2<Long, MatrixBlock>>> pfs = partition(numWorkers, rblkID, features);
+		List<Tuple2<Integer, Tuple2<Long, MatrixBlock>>> pls = partition(numWorkers, rblkID, labels);
+		return new Result(pfs, pls);
+	}
+
+	private List<Tuple2<Integer, Tuple2<Long, MatrixBlock>>> partition(int numWorkers, int rblkID, MatrixBlock mb) {
+		return IntStream.range(0, numWorkers).mapToObj(i -> i).flatMap(workerID -> {
+				MatrixBlock partialPerm = _globalPerms.get(workerID).getBlock(rblkID, 1);
+				return IntStream.range(0, mb.getNumRows()).mapToObj(r -> {
+					MatrixBlock rowMB = ParamservUtils.sliceMatrixBlock(mb, r + 1, r + 1);
+					long shiftedPosition = (long) partialPerm.getValue(r, 0);
+					return new Tuple2<>(workerID, new Tuple2<>(shiftedPosition, rowMB));
+				});
+			}).collect(Collectors.toList());
+	}
+}

http://git-wip-us.apache.org/repos/asf/systemml/blob/8def2040/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/SparkDataPartitioner.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/SparkDataPartitioner.java b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/SparkDataPartitioner.java
new file mode 100644
index 0000000..6883d0f
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/SparkDataPartitioner.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.controlprogram.paramserv.spark;
+
+import static org.apache.sysml.runtime.controlprogram.paramserv.ParamservUtils.SEED;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.apache.sysml.parser.Statement;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.controlprogram.paramserv.ParamservUtils;
+import org.apache.sysml.runtime.instructions.spark.data.PartitionedBroadcast;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.util.DataConverter;
+
+public class SparkDataPartitioner implements Serializable {
+
+	private static final long serialVersionUID = 6841548626711057448L;
+	private DataPartitionSparkScheme _scheme;
+
+	protected SparkDataPartitioner(Statement.PSScheme scheme, SparkExecutionContext sec, int numEntries, int numWorkers) {
+		switch (scheme) {
+			case DISJOINT_CONTIGUOUS:
+				_scheme = new DCSparkScheme();
+				// Create the worker id indicator
+				createDCIndicator(sec, numWorkers, numEntries);
+				break;
+			case DISJOINT_ROUND_ROBIN:
+				_scheme = new DRRSparkScheme();
+				// Create the worker id indicator
+				createDRIndicator(sec, numWorkers, numEntries);
+				break;
+			case DISJOINT_RANDOM:
+				_scheme = new DRSparkScheme();
+				// Create the global permutation
+				createGlobalPermutations(sec, numEntries, 1);
+				// Create the worker id indicator
+				createDCIndicator(sec, numWorkers, numEntries);
+				break;
+			case OVERLAP_RESHUFFLE:
+				_scheme = new ORSparkScheme();
+				// Create the global permutation seperately for each worker
+				createGlobalPermutations(sec, numEntries, numWorkers);
+				break;
+		}
+	}
+
+	private void createDRIndicator(SparkExecutionContext sec, int numWorkers, int numEntries) {
+		double[] vector = IntStream.range(0, numEntries).mapToDouble(n -> n % numWorkers).toArray();
+		MatrixBlock vectorMB = DataConverter.convertToMatrixBlock(vector, true);
+		_scheme.setWorkerIndicator(sec.getBroadcastForMatrixObject(ParamservUtils.newMatrixObject(vectorMB)));
+	}
+
+	private void createDCIndicator(SparkExecutionContext sec, int numWorkers, int numEntries) {
+		double[] vector = new double[numEntries];
+		int batchSize = (int) Math.ceil((double) numEntries / numWorkers);
+		for (int i = 1; i < numWorkers; i++) {
+			int begin = batchSize * i;
+			int end = Math.min(begin + batchSize, numEntries);
+			Arrays.fill(vector, begin, end, i);
+		}
+		MatrixBlock vectorMB = DataConverter.convertToMatrixBlock(vector, true);
+		_scheme.setWorkerIndicator(sec.getBroadcastForMatrixObject(ParamservUtils.newMatrixObject(vectorMB)));
+	}
+
+	private void createGlobalPermutations(SparkExecutionContext sec, int numEntries, int numPerm) {
+		List<PartitionedBroadcast<MatrixBlock>> perms = IntStream.range(0, numPerm).mapToObj(i -> {
+			MatrixBlock perm = MatrixBlock.sampleOperations(numEntries, numEntries, false, SEED+i);
+			// Create the source-target id vector from the permutation ranging from 1 to number of entries
+			double[] vector = new double[numEntries];
+			for (int j = 0; j < perm.getDenseBlockValues().length; j++) {
+				vector[(int) perm.getDenseBlockValues()[j] - 1] = j;
+			}
+			MatrixBlock vectorMB = DataConverter.convertToMatrixBlock(vector, true);
+			return sec.getBroadcastForMatrixObject(ParamservUtils.newMatrixObject(vectorMB));
+		}).collect(Collectors.toList());
+		_scheme.setGlobalPermutation(perms);
+	}
+
+	public DataPartitionSparkScheme.Result doPartitioning(int numWorkers, MatrixBlock features, MatrixBlock labels,
+			long rowID) {
+		// Set the rowID in order to get the according permutation
+		return _scheme.doPartitioning(numWorkers, (int) rowID, features, labels);
+	}
+}

http://git-wip-us.apache.org/repos/asf/systemml/blob/8def2040/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/SparkPSWorker.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/SparkPSWorker.java b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/SparkPSWorker.java
new file mode 100644
index 0000000..69da56c
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/spark/SparkPSWorker.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.controlprogram.paramserv.spark;
+
+import java.io.Serializable;
+
+import org.apache.spark.api.java.function.VoidFunction;
+import org.apache.sysml.parser.Statement;
+import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.paramserv.ParamServer;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+
+import scala.Tuple2;
+
+public class SparkPSWorker implements VoidFunction<Tuple2<Integer, Tuple2<MatrixBlock, MatrixBlock>>>, Serializable {
+
+	private static final long serialVersionUID = -8674739573419648732L;
+
+	public SparkPSWorker() {
+		// No-args constructor used for deserialization
+	}
+
+	public SparkPSWorker(String updFunc, Statement.PSFrequency freq, int epochs, long batchSize,
+			MatrixObject valFeatures, MatrixObject valLabels, ExecutionContext ec, ParamServer ps) {
+	}
+
+	@Override
+	public void call(Tuple2<Integer, Tuple2<MatrixBlock, MatrixBlock>> input) throws Exception {
+	}
+}

http://git-wip-us.apache.org/repos/asf/systemml/blob/8def2040/src/main/java/org/apache/sysml/runtime/instructions/cp/ParamservBuiltinCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ParamservBuiltinCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ParamservBuiltinCPInstruction.java
index 25bc113..3d61625 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ParamservBuiltinCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ParamservBuiltinCPInstruction.java
@@ -58,15 +58,14 @@ import org.apache.sysml.runtime.DMLRuntimeException;
 import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
 import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
 import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.controlprogram.paramserv.DataPartitionScheme;
 import org.apache.sysml.runtime.controlprogram.paramserv.DataPartitioner;
-import org.apache.sysml.runtime.controlprogram.paramserv.DataPartitionerDC;
-import org.apache.sysml.runtime.controlprogram.paramserv.DataPartitionerDR;
-import org.apache.sysml.runtime.controlprogram.paramserv.DataPartitionerDRR;
-import org.apache.sysml.runtime.controlprogram.paramserv.DataPartitionerOR;
 import org.apache.sysml.runtime.controlprogram.paramserv.LocalPSWorker;
 import org.apache.sysml.runtime.controlprogram.paramserv.LocalParamServer;
 import org.apache.sysml.runtime.controlprogram.paramserv.ParamServer;
 import org.apache.sysml.runtime.controlprogram.paramserv.ParamservUtils;
+import org.apache.sysml.runtime.controlprogram.paramserv.spark.SparkPSWorker;
 import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
 import org.apache.sysml.runtime.controlprogram.parfor.stat.Timing;
 import org.apache.sysml.runtime.matrix.operators.Operator;
@@ -97,9 +96,32 @@ public class ParamservBuiltinCPInstruction extends ParameterizedBuiltinCPInstruc
 
 	@Override
 	public void processInstruction(ExecutionContext ec) {
-		Timing tSetup = DMLScript.STATISTICS ? new Timing(true) : null;
-
 		PSModeType mode = getPSMode();
+		switch (mode) {
+			case LOCAL:
+				runLocally(ec, mode);
+				break;
+			case REMOTE_SPARK:
+				runOnSpark((SparkExecutionContext) ec, mode);
+				break;
+		}
+	}
+
+	private void runOnSpark(SparkExecutionContext sec, PSModeType mode) {
+		PSScheme scheme = getScheme();
+		int workerNum = getWorkerNum(mode);
+
+		MatrixObject features = sec.getMatrixObject(getParam(PS_FEATURES));
+		MatrixObject labels = sec.getMatrixObject(getParam(PS_LABELS));
+
+		SparkPSWorker worker = new SparkPSWorker();
+		ParamservUtils.doPartitionOnSpark(sec, features, labels, scheme, workerNum) // Do data partitioning
+				.foreach(worker);   // Run remote workers
+
+	}
+
+	private void runLocally(ExecutionContext ec, PSModeType mode) {
+		Timing tSetup = DMLScript.STATISTICS ? new Timing(true) : null;
 		int workerNum = getWorkerNum(mode);
 		BasicThreadFactory factory = new BasicThreadFactory.Builder()
 			.namingPattern("workers-pool-thread-%d").build();
@@ -132,13 +154,13 @@ public class ParamservBuiltinCPInstruction extends ParameterizedBuiltinCPInstruc
 		MatrixObject valFeatures = ec.getMatrixObject(getParam(PS_VAL_FEATURES));
 		MatrixObject valLabels = ec.getMatrixObject(getParam(PS_VAL_LABELS));
 		List<LocalPSWorker> workers = IntStream.range(0, workerNum)
-			.mapToObj(i -> new LocalPSWorker(i, updFunc, freq, epochs, getBatchSize(), valFeatures, valLabels, workerECs.get(i), ps))
-			.collect(Collectors.toList());
+		   .mapToObj(i -> new LocalPSWorker(i, updFunc, freq, epochs, getBatchSize(), valFeatures, valLabels, workerECs.get(i), ps))
+		   .collect(Collectors.toList());
 
 		// Do data partition
 		PSScheme scheme = getScheme();
-		doDataPartitioning(scheme, ec, workers);
-		
+		partitionLocally(scheme, ec, workers);
+
 		if (DMLScript.STATISTICS)
 			Statistics.accPSSetupTime((long) tSetup.stop());
 
@@ -180,8 +202,6 @@ public class ParamservBuiltinCPInstruction extends ParameterizedBuiltinCPInstruc
 		} catch (IllegalArgumentException e) {
 			throw new DMLRuntimeException(String.format("Paramserv function: not support ps execution mode '%s'", getParam(PS_MODE)));
 		}
-		if( mode == PSModeType.REMOTE_SPARK )
-			throw new DMLRuntimeException("Do not support remote spark.");
 		return mode;
 	}
 
@@ -236,12 +256,13 @@ public class ParamservBuiltinCPInstruction extends ParameterizedBuiltinCPInstruc
 		switch (mode) {
 			case LOCAL:
 				// default worker number: available cores - 1 (assign one process for agg service)
-				int workerNum = getRemainingCores();
-				if (getParameterMap().containsKey(PS_PARALLELISM))
-					workerNum = Integer.valueOf(getParam(PS_PARALLELISM));
-				return workerNum;
+				return getParameterMap().containsKey(PS_PARALLELISM) ?
+					Integer.valueOf(getParam(PS_PARALLELISM)) : getRemainingCores();
+			case REMOTE_SPARK:
+				return getParameterMap().containsKey(PS_PARALLELISM) ?
+					Integer.valueOf(getParam(PS_PARALLELISM)) : SparkExecutionContext.getDefaultParallelism(true);
 			default:
-				throw new DMLRuntimeException("Unsupported parameter server: "+mode.name());
+				throw new DMLRuntimeException("Unsupported parameter server: " + mode.name());
 		}
 	}
 
@@ -279,22 +300,24 @@ public class ParamservBuiltinCPInstruction extends ParameterizedBuiltinCPInstruc
 		return hyperparams;
 	}
 
-	private void doDataPartitioning(PSScheme scheme, ExecutionContext ec, List<LocalPSWorker> workers) {
+	private void partitionLocally(PSScheme scheme, ExecutionContext ec, List<LocalPSWorker> workers) {
 		MatrixObject features = ec.getMatrixObject(getParam(PS_FEATURES));
 		MatrixObject labels = ec.getMatrixObject(getParam(PS_LABELS));
-		switch (scheme) {
-			case DISJOINT_CONTIGUOUS:
-				doDataPartitioning(new DataPartitionerDC(), features, labels, workers);
-				break;
-			case DISJOINT_ROUND_ROBIN:
-				doDataPartitioning(new DataPartitionerDRR(), features, labels, workers);
-				break;
-			case DISJOINT_RANDOM:
-				doDataPartitioning(new DataPartitionerDR(), features, labels, workers);
-				break;
-			case OVERLAP_RESHUFFLE:
-				doDataPartitioning(new DataPartitionerOR(), features, labels, workers);
-				break;
+		DataPartitionScheme.Result result = new DataPartitioner(scheme).doPartitioning(workers.size(), features.acquireRead(), labels.acquireRead());
+		features.release();
+		labels.release();
+		List<MatrixObject> pfs = result.pFeatures;
+		List<MatrixObject> pls = result.pLabels;
+		if (pfs.size() < workers.size()) {
+			if (LOG.isWarnEnabled()) {
+				LOG.warn(String.format("There is only %d batches of data but has %d workers. "
+					+ "Hence, reset the number of workers with %d.", pfs.size(), workers.size(), pfs.size()));
+			}
+			workers = workers.subList(0, pfs.size());
+		}
+		for (int i = 0; i < workers.size(); i++) {
+			workers.get(i).setFeatures(pfs.get(i));
+			workers.get(i).setLabels(pls.get(i));
 		}
 	}
 
@@ -310,20 +333,4 @@ public class ParamservBuiltinCPInstruction extends ParameterizedBuiltinCPInstruc
 		return scheme;
 	}
 
-	private void doDataPartitioning(DataPartitioner dp, MatrixObject features, MatrixObject labels, List<LocalPSWorker> workers) {
-		DataPartitioner.Result result = dp.doPartitioning(workers.size(), features, labels);
-		List<MatrixObject> pfs = result.pFeatures;
-		List<MatrixObject> pls = result.pLabels;
-		if (pfs.size() < workers.size()) {
-			if (LOG.isWarnEnabled()) {
-				LOG.warn(String.format("There is only %d batches of data but has %d workers. "
-					+ "Hence, reset the number of workers with %d.", pfs.size(), workers.size(), pfs.size()));
-			}
-			workers = workers.subList(0, pfs.size());
-		}
-		for (int i = 0; i < workers.size(); i++) {
-			workers.get(i).setFeatures(pfs.get(i));
-			workers.get(i).setLabels(pls.get(i));
-		}
-	}
 }

http://git-wip-us.apache.org/repos/asf/systemml/blob/8def2040/src/test/java/org/apache/sysml/test/integration/functions/paramserv/BaseDataPartitionerTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/paramserv/BaseDataPartitionerTest.java b/src/test/java/org/apache/sysml/test/integration/functions/paramserv/BaseDataPartitionerTest.java
new file mode 100644
index 0000000..0092aed
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/paramserv/BaseDataPartitionerTest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.paramserv;
+
+import java.util.stream.IntStream;
+
+import org.apache.sysml.parser.Statement;
+import org.apache.sysml.runtime.controlprogram.paramserv.DataPartitionScheme;
+import org.apache.sysml.runtime.controlprogram.paramserv.DataPartitioner;
+import org.apache.sysml.runtime.controlprogram.paramserv.ParamservUtils;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.util.DataConverter;
+
+public abstract class BaseDataPartitionerTest {
+
+	protected static final int ROW_SIZE = 4000;
+	protected static final int COL_SIZE = 2000;
+	protected static final int WORKER_NUM = 4;
+
+	protected MatrixBlock[] generateData() {
+		double[][] df = new double[BaseDataPartitionerTest.ROW_SIZE][BaseDataPartitionerTest.COL_SIZE];
+		for (int i = 0; i < BaseDataPartitionerTest.ROW_SIZE; i++) {
+			for (int j = 0; j < BaseDataPartitionerTest.COL_SIZE; j++) {
+				df[i][j] = i * BaseDataPartitionerTest.COL_SIZE + j;
+			}
+		}
+		double[] dl = new double[BaseDataPartitionerTest.ROW_SIZE];
+		for (int i = 0; i < BaseDataPartitionerTest.ROW_SIZE; i++) {
+			dl[i] = i;
+		}
+		MatrixBlock fmb = DataConverter.convertToMatrixBlock(df);
+		MatrixBlock lmb = DataConverter.convertToMatrixBlock(dl, true);
+		return new MatrixBlock[] { fmb, lmb };
+	}
+
+	protected double[] generateExpectedData(int from, int to) {
+		return IntStream.range(from, to).mapToDouble(i -> (double) i).toArray();
+	}
+
+	protected DataPartitionScheme.Result launchLocalDataPartitionerDC() {
+		DataPartitioner dp = new DataPartitioner(Statement.PSScheme.DISJOINT_CONTIGUOUS);
+		MatrixBlock[] mbs = generateData();
+		return dp.doPartitioning(WORKER_NUM, mbs[0], mbs[1]);
+	}
+
+	protected DataPartitionScheme.Result launchLocalDataPartitionerDR(MatrixBlock[] mbs) {
+		ParamservUtils.SEED = System.nanoTime();
+		DataPartitioner dp = new DataPartitioner(Statement.PSScheme.DISJOINT_RANDOM);
+		return dp.doPartitioning(WORKER_NUM, mbs[0], mbs[1]);
+	}
+
+	protected DataPartitionScheme.Result launchLocalDataPartitionerDRR() {
+		DataPartitioner dp = new DataPartitioner(Statement.PSScheme.DISJOINT_ROUND_ROBIN);
+		MatrixBlock[] mbs = generateData();
+		return dp.doPartitioning(WORKER_NUM, mbs[0], mbs[1]);
+	}
+
+	protected DataPartitionScheme.Result launchLocalDataPartitionerOR() {
+		DataPartitioner dp = new DataPartitioner(Statement.PSScheme.OVERLAP_RESHUFFLE);
+		MatrixBlock[] mbs = generateData();
+		return dp.doPartitioning(WORKER_NUM, mbs[0], mbs[1]);
+	}
+}