You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2013/09/26 22:49:16 UTC
[1/7] git commit: Start of working on SPARK-615
Updated Branches:
refs/heads/master 560ee5c9b -> c514cd158
Start of working on SPARK-615
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/74f710f6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/74f710f6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/74f710f6
Branch: refs/heads/master
Commit: 74f710f6cda31c1489e8f0cc130021ce4e9e60c6
Parents: a310de6
Author: Holden Karau <ho...@pigscanfly.ca>
Authored: Wed Sep 11 22:35:58 2013 -0700
Committer: Holden Karau <ho...@pigscanfly.ca>
Committed: Wed Sep 11 22:35:58 2013 -0700
----------------------------------------------------------------------
.../main/scala/org/apache/spark/api/java/JavaRDDLike.scala | 8 ++++++++
1 file changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/74f710f6/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala b/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
index 7e6e691..9ad175e 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
@@ -68,6 +68,14 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable {
new JavaRDD(rdd.map(f)(f.returnType()))(f.returnType())
/**
+ * Return a new RDD by applying a function to each partition of this RDD, while tracking the index
+ * of the original partition.
+ */
+ def mapPartitionsWithIndex(f: JFunction2[Int, T, R],
+ preservesPartitioning: Boolean = false): JavaRDD[R] =
+ new JavaRDD(MapPartitionsWithIndexRDD(this, sc.clean(f), preservesPartitioning))
+
+ /**
* Return a new RDD by applying a function to all elements of this RDD.
*/
def map[R](f: DoubleFunction[T]): JavaDoubleRDD =
[2/7] git commit: Make mapPartitionsWithIndex work with JavaRDD's
Posted by rx...@apache.org.
Make mapPartitionsWithIndex work with JavaRDD's
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/bfcddf47
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/bfcddf47
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/bfcddf47
Branch: refs/heads/master
Commit: bfcddf4700023f53d5eed92ef8ef75c072af3ced
Parents: 74f710f
Author: Holden Karau <ho...@pigscanfly.ca>
Authored: Sat Sep 14 15:53:42 2013 -0700
Committer: Holden Karau <ho...@pigscanfly.ca>
Committed: Sat Sep 14 15:53:42 2013 -0700
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/bfcddf47/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala b/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
index 9ad175e..264c4bc 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
@@ -71,9 +71,10 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable {
* Return a new RDD by applying a function to each partition of this RDD, while tracking the index
* of the original partition.
*/
- def mapPartitionsWithIndex(f: JFunction2[Int, T, R],
+ def mapPartitionsWithIndex[R: ClassManifest](f: JFunction2[Int, java.util.Iterator[T], java.util.Iterator[R]],
preservesPartitioning: Boolean = false): JavaRDD[R] =
- new JavaRDD(MapPartitionsWithIndexRDD(this, sc.clean(f), preservesPartitioning))
+ new JavaRDD(rdd.mapPartitionsWithIndex(((a,b) => f(a,asJavaIterator(b))),
+ preservesPartitioning))
/**
* Return a new RDD by applying a function to all elements of this RDD.
[4/7] git commit: Fix build on ubuntu
Posted by rx...@apache.org.
Fix build on ubuntu
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/68068977
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/68068977
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/68068977
Branch: refs/heads/master
Commit: 68068977b85d2355223e21ebf4e546a13f0a8585
Parents: 452db10
Author: Holden Karau <ho...@pigscanfly.ca>
Authored: Sat Sep 14 20:51:11 2013 -0700
Committer: Holden Karau <ho...@pigscanfly.ca>
Committed: Sat Sep 14 20:51:11 2013 -0700
----------------------------------------------------------------------
project/SparkBuild.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/68068977/project/SparkBuild.scala
----------------------------------------------------------------------
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 77e211c..eb5a893 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -81,7 +81,7 @@ object SparkBuild extends Build {
organization := "org.apache.spark",
version := "0.8.0-SNAPSHOT",
scalaVersion := "2.9.3",
- scalacOptions := Seq("-unchecked", "-optimize", "-deprecation",
+ scalacOptions := Seq("-Xmax-classfile-name", "120", "-unchecked", "-optimize", "-deprecation",
"-target:" + SCALAC_JVM_VERSION),
javacOptions := Seq("-target", JAVAC_JVM_VERSION, "-source", JAVAC_JVM_VERSION),
unmanagedJars in Compile <<= baseDirectory map { base => (base / "lib" ** "*.jar").classpath },
[5/7] git commit: Switch indent from 2 to 4 spaces
Posted by rx...@apache.org.
Switch indent from 2 to 4 spaces
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/7fe0b0ff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/7fe0b0ff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/7fe0b0ff
Branch: refs/heads/master
Commit: 7fe0b0ff568e8cf43efaabf46ac760cc8b91cf79
Parents: 6806897
Author: Holden Karau <ho...@pigscanfly.ca>
Authored: Sun Sep 22 19:44:51 2013 -0700
Committer: Holden Karau <ho...@pigscanfly.ca>
Committed: Sun Sep 22 19:44:51 2013 -0700
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/7fe0b0ff/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala b/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
index 264c4bc..b932c7c 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
@@ -73,8 +73,8 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable {
*/
def mapPartitionsWithIndex[R: ClassManifest](f: JFunction2[Int, java.util.Iterator[T], java.util.Iterator[R]],
preservesPartitioning: Boolean = false): JavaRDD[R] =
- new JavaRDD(rdd.mapPartitionsWithIndex(((a,b) => f(a,asJavaIterator(b))),
- preservesPartitioning))
+ new JavaRDD(rdd.mapPartitionsWithIndex(((a,b) => f(a,asJavaIterator(b))),
+ preservesPartitioning))
/**
* Return a new RDD by applying a function to all elements of this RDD.
[3/7] git commit: Merge branch 'master' of
https://github.com/mesos/spark
Posted by rx...@apache.org.
Merge branch 'master' of https://github.com/mesos/spark
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/452db108
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/452db108
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/452db108
Branch: refs/heads/master
Commit: 452db1083c956b9f776d09d2c4fff1d93a428907
Parents: bfcddf4 c4c1db2
Author: Holden Karau <ho...@pigscanfly.ca>
Authored: Sat Sep 14 15:54:04 2013 -0700
Committer: Holden Karau <ho...@pigscanfly.ca>
Committed: Sat Sep 14 15:54:04 2013 -0700
----------------------------------------------------------------------
pom.xml | 12 +++++++-----
yarn/pom.xml | 5 +++++
2 files changed, 12 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
[6/7] git commit: Fix formatting :)
Posted by rx...@apache.org.
Fix formatting :)
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/0cef6835
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/0cef6835
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/0cef6835
Branch: refs/heads/master
Commit: 0cef683553414ba880d90527cc5f37e119efc782
Parents: 7fe0b0f
Author: Holden Karau <ho...@pigscanfly.ca>
Authored: Mon Sep 23 19:39:42 2013 -0700
Committer: Holden Karau <ho...@pigscanfly.ca>
Committed: Mon Sep 23 19:39:42 2013 -0700
----------------------------------------------------------------------
.../main/scala/org/apache/spark/api/java/JavaRDDLike.scala | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/0cef6835/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala b/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
index b932c7c..7a3568c 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
@@ -71,10 +71,11 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable {
* Return a new RDD by applying a function to each partition of this RDD, while tracking the index
* of the original partition.
*/
- def mapPartitionsWithIndex[R: ClassManifest](f: JFunction2[Int, java.util.Iterator[T], java.util.Iterator[R]],
- preservesPartitioning: Boolean = false): JavaRDD[R] =
- new JavaRDD(rdd.mapPartitionsWithIndex(((a,b) => f(a,asJavaIterator(b))),
- preservesPartitioning))
+ def mapPartitionsWithIndex[R: ClassManifest](
+ f: JFunction2[Int, java.util.Iterator[T], java.util.Iterator[R]],
+ preservesPartitioning: Boolean = false): JavaRDD[R] =
+ new JavaRDD(rdd.mapPartitionsWithIndex(((a,b) => f(a,asJavaIterator(b))),
+ preservesPartitioning))
/**
* Return a new RDD by applying a function to all elements of this RDD.
[7/7] git commit: Merge pull request #930 from holdenk/master
Posted by rx...@apache.org.
Merge pull request #930 from holdenk/master
Add mapPartitionsWithIndex
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/c514cd15
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/c514cd15
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/c514cd15
Branch: refs/heads/master
Commit: c514cd15879d510406712b1cebdaf0c0db9151c9
Parents: 560ee5c 0cef683
Author: Reynold Xin <rx...@apache.org>
Authored: Thu Sep 26 13:48:20 2013 -0700
Committer: Reynold Xin <rx...@apache.org>
Committed: Thu Sep 26 13:48:20 2013 -0700
----------------------------------------------------------------------
.../scala/org/apache/spark/api/java/JavaRDDLike.scala | 10 ++++++++++
project/SparkBuild.scala | 2 +-
2 files changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/c514cd15/project/SparkBuild.scala
----------------------------------------------------------------------
diff --cc project/SparkBuild.scala
index a2e2959,eb5a893..99cdadb
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@@ -79,9 -79,9 +79,9 @@@ object SparkBuild extends Build
def sharedSettings = Defaults.defaultSettings ++ Seq(
organization := "org.apache.spark",
- version := "0.8.0-SNAPSHOT",
+ version := "0.9.0-incubating-SNAPSHOT",
scalaVersion := "2.9.3",
- scalacOptions := Seq("-unchecked", "-optimize", "-deprecation",
+ scalacOptions := Seq("-Xmax-classfile-name", "120", "-unchecked", "-optimize", "-deprecation",
"-target:" + SCALAC_JVM_VERSION),
javacOptions := Seq("-target", JAVAC_JVM_VERSION, "-source", JAVAC_JVM_VERSION),
unmanagedJars in Compile <<= baseDirectory map { base => (base / "lib" ** "*.jar").classpath },