You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by gw...@apache.org on 2016/11/29 19:21:02 UTC
incubator-systemml git commit: [SYSTEMML-874] Return
LazyIterableIterator in PairFlatMapFunction call
Repository: incubator-systemml
Updated Branches:
refs/heads/master ce71ab16a -> d1f6ba389
[SYSTEMML-874] Return LazyIterableIterator in PairFlatMapFunction call
Support both Iterable and Iterator in call method override of
PairFlatMapFunction for classes already using LazyIterableIterator.
This maintains backward compatibility between different Spark versions.
Closes #296.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/d1f6ba38
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/d1f6ba38
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/d1f6ba38
Branch: refs/heads/master
Commit: d1f6ba389bd23975b74c49711c81f9d0fdf76ffb
Parents: ce71ab1
Author: Glenn Weidner <gw...@us.ibm.com>
Authored: Tue Nov 29 11:14:46 2016 -0800
Committer: Glenn Weidner <gw...@us.ibm.com>
Committed: Tue Nov 29 11:14:47 2016 -0800
----------------------------------------------------------------------
.../runtime/instructions/spark/FrameAppendMSPInstruction.java | 2 +-
.../runtime/instructions/spark/FrameIndexingSPInstruction.java | 4 ++--
.../sysml/runtime/instructions/spark/MapmmSPInstruction.java | 2 +-
.../runtime/instructions/spark/MatrixAppendMSPInstruction.java | 2 +-
.../runtime/instructions/spark/MatrixIndexingSPInstruction.java | 4 ++--
.../runtime/instructions/spark/QuaternarySPInstruction.java | 2 +-
.../runtime/instructions/spark/UaggOuterChainSPInstruction.java | 4 ++--
.../instructions/spark/functions/CopyBlockPairFunction.java | 2 +-
.../spark/functions/MatrixVectorBinaryOpPartitionFunction.java | 2 +-
.../spark/functions/OuterVectorBinaryOpFunction.java | 2 +-
10 files changed, 13 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d1f6ba38/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameAppendMSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameAppendMSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameAppendMSPInstruction.java
index c7be957..218a241 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameAppendMSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameAppendMSPInstruction.java
@@ -94,7 +94,7 @@ public class FrameAppendMSPInstruction extends AppendMSPInstruction
}
@Override
- public Iterable<Tuple2<Long, FrameBlock>> call(Iterator<Tuple2<Long, FrameBlock>> arg0)
+ public LazyIterableIterator<Tuple2<Long, FrameBlock>> call(Iterator<Tuple2<Long, FrameBlock>> arg0)
throws Exception
{
return new MapAppendPartitionIterator(arg0);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d1f6ba38/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameIndexingSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameIndexingSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameIndexingSPInstruction.java
index 5082ddd..6e25446 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameIndexingSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameIndexingSPInstruction.java
@@ -287,7 +287,7 @@ public class FrameIndexingSPInstruction extends IndexingSPInstruction
}
@Override
- public Iterable<Tuple2<Long, FrameBlock>> call(Iterator<Tuple2<Long, FrameBlock>> arg0)
+ public LazyIterableIterator<Tuple2<Long, FrameBlock>> call(Iterator<Tuple2<Long, FrameBlock>> arg0)
throws Exception
{
return new LeftIndexPartitionIterator(arg0);
@@ -391,7 +391,7 @@ public class FrameIndexingSPInstruction extends IndexingSPInstruction
}
@Override
- public Iterable<Tuple2<Long, FrameBlock>> call(Iterator<Tuple2<Long, FrameBlock>> arg0)
+ public LazyIterableIterator<Tuple2<Long, FrameBlock>> call(Iterator<Tuple2<Long, FrameBlock>> arg0)
throws Exception
{
return new SliceBlockPartitionIterator(arg0);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d1f6ba38/src/main/java/org/apache/sysml/runtime/instructions/spark/MapmmSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MapmmSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MapmmSPInstruction.java
index 7a7685e..dfe8791 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MapmmSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MapmmSPInstruction.java
@@ -348,7 +348,7 @@ public class MapmmSPInstruction extends BinarySPInstruction
}
@Override
- public Iterable<Tuple2<MatrixIndexes, MatrixBlock>> call(Iterator<Tuple2<MatrixIndexes, MatrixBlock>> arg0)
+ public LazyIterableIterator<Tuple2<MatrixIndexes, MatrixBlock>> call(Iterator<Tuple2<MatrixIndexes, MatrixBlock>> arg0)
throws Exception
{
return new MapMMPartitionIterator(arg0);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d1f6ba38/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixAppendMSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixAppendMSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixAppendMSPInstruction.java
index 4bce11f..7100011 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixAppendMSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixAppendMSPInstruction.java
@@ -206,7 +206,7 @@ public class MatrixAppendMSPInstruction extends AppendMSPInstruction
}
@Override
- public Iterable<Tuple2<MatrixIndexes, MatrixBlock>> call(Iterator<Tuple2<MatrixIndexes, MatrixBlock>> arg0)
+ public LazyIterableIterator<Tuple2<MatrixIndexes, MatrixBlock>> call(Iterator<Tuple2<MatrixIndexes, MatrixBlock>> arg0)
throws Exception
{
return new MapAppendPartitionIterator(arg0);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d1f6ba38/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java
index 9fbb6e1..8d4d296 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java
@@ -359,7 +359,7 @@ public class MatrixIndexingSPInstruction extends IndexingSPInstruction
}
@Override
- public Iterable<Tuple2<MatrixIndexes, MatrixBlock>> call(Iterator<Tuple2<MatrixIndexes, MatrixBlock>> arg0)
+ public LazyIterableIterator<Tuple2<MatrixIndexes, MatrixBlock>> call(Iterator<Tuple2<MatrixIndexes, MatrixBlock>> arg0)
throws Exception
{
return new LeftIndexPartitionIterator(arg0);
@@ -473,7 +473,7 @@ public class MatrixIndexingSPInstruction extends IndexingSPInstruction
}
@Override
- public Iterable<Tuple2<MatrixIndexes, MatrixBlock>> call(Iterator<Tuple2<MatrixIndexes, MatrixBlock>> arg0)
+ public LazyIterableIterator<Tuple2<MatrixIndexes, MatrixBlock>> call(Iterator<Tuple2<MatrixIndexes, MatrixBlock>> arg0)
throws Exception
{
return new SliceBlockPartitionIterator(arg0);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d1f6ba38/src/main/java/org/apache/sysml/runtime/instructions/spark/QuaternarySPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/QuaternarySPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/QuaternarySPInstruction.java
index 66a0e24..063b6e7 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/QuaternarySPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/QuaternarySPInstruction.java
@@ -374,7 +374,7 @@ public class QuaternarySPInstruction extends ComputationSPInstruction
}
@Override
- public Iterable<Tuple2<MatrixIndexes, MatrixBlock>> call(Iterator<Tuple2<MatrixIndexes, MatrixBlock>> arg)
+ public LazyIterableIterator<Tuple2<MatrixIndexes, MatrixBlock>> call(Iterator<Tuple2<MatrixIndexes, MatrixBlock>> arg)
throws Exception
{
return new RDDQuaternaryPartitionIterator(arg);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d1f6ba38/src/main/java/org/apache/sysml/runtime/instructions/spark/UaggOuterChainSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/UaggOuterChainSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/UaggOuterChainSPInstruction.java
index 88c9ded..d6cc8e8 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/UaggOuterChainSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/UaggOuterChainSPInstruction.java
@@ -248,7 +248,7 @@ public class UaggOuterChainSPInstruction extends BinarySPInstruction
}
@Override
- public Iterable<Tuple2<MatrixIndexes, MatrixBlock>> call(Iterator<Tuple2<MatrixIndexes, MatrixBlock>> arg0)
+ public LazyIterableIterator<Tuple2<MatrixIndexes, MatrixBlock>> call(Iterator<Tuple2<MatrixIndexes, MatrixBlock>> arg0)
throws Exception
{
return new RDDMapUAggOuterChainIterator(arg0);
@@ -320,7 +320,7 @@ public class UaggOuterChainSPInstruction extends BinarySPInstruction
}
@Override
- public Iterable<Tuple2<MatrixIndexes, MatrixBlock>> call(Iterator<Tuple2<MatrixIndexes, MatrixBlock>> arg)
+ public LazyIterableIterator<Tuple2<MatrixIndexes, MatrixBlock>> call(Iterator<Tuple2<MatrixIndexes, MatrixBlock>> arg)
throws Exception
{
return new RDDMapGenUAggOuterChainIterator(arg);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d1f6ba38/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyBlockPairFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyBlockPairFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyBlockPairFunction.java
index aa8996f..42d4843 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyBlockPairFunction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyBlockPairFunction.java
@@ -51,7 +51,7 @@ public class CopyBlockPairFunction implements PairFlatMapFunction<Iterator<Tuple
}
@Override
- public Iterable<Tuple2<MatrixIndexes, MatrixBlock>> call(Iterator<Tuple2<MatrixIndexes, MatrixBlock>> arg0)
+ public LazyIterableIterator<Tuple2<MatrixIndexes, MatrixBlock>> call(Iterator<Tuple2<MatrixIndexes, MatrixBlock>> arg0)
throws Exception
{
return new CopyBlockPairIterator(arg0);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d1f6ba38/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpPartitionFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpPartitionFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpPartitionFunction.java
index 597c028..70f3318 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpPartitionFunction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpPartitionFunction.java
@@ -48,7 +48,7 @@ public class MatrixVectorBinaryOpPartitionFunction implements PairFlatMapFunctio
}
@Override
- public Iterable<Tuple2<MatrixIndexes, MatrixBlock>> call(Iterator<Tuple2<MatrixIndexes, MatrixBlock>> arg0)
+ public LazyIterableIterator<Tuple2<MatrixIndexes, MatrixBlock>> call(Iterator<Tuple2<MatrixIndexes, MatrixBlock>> arg0)
throws Exception
{
return new MapBinaryPartitionIterator( arg0 );
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d1f6ba38/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/OuterVectorBinaryOpFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/OuterVectorBinaryOpFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/OuterVectorBinaryOpFunction.java
index e98d936..ddb77e8 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/OuterVectorBinaryOpFunction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/OuterVectorBinaryOpFunction.java
@@ -44,7 +44,7 @@ public class OuterVectorBinaryOpFunction implements PairFlatMapFunction<Tuple2<M
}
@Override
- public Iterable<Tuple2<MatrixIndexes, MatrixBlock>> call(Tuple2<MatrixIndexes, MatrixBlock> arg0)
+ public OuterVectorBinaryOpIterator call(Tuple2<MatrixIndexes, MatrixBlock> arg0)
throws Exception
{
return new OuterVectorBinaryOpIterator(arg0);