You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2016/01/12 17:11:23 UTC
incubator-tinkerpop git commit: InputRDD and OutputRDD have default
methods for read/writeMemoryRDD which return empty RDDs and Iterators. Most
providers will never implement these methods as they will simply store their
data using XXXFormatRDD or Persis
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/master 6be9ce075 -> 1d36fea23
InputRDD and OutputRDD have default methods for read/writeMemoryRDD which return empty RDDs and Iterators. Most providers will never implement these methods as they will simply store their data using XXXFormatRDD or PersistedXXXRDD. Thus, default methods make it easier for providers.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/1d36fea2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/1d36fea2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/1d36fea2
Branch: refs/heads/master
Commit: 1d36fea23777d05a7e600f37bcb9bc0873b400f8
Parents: 6be9ce0
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Jan 12 09:11:10 2016 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue Jan 12 09:11:19 2016 -0700
----------------------------------------------------------------------
.../apache/tinkerpop/gremlin/spark/structure/io/InputRDD.java | 6 +++++-
.../tinkerpop/gremlin/spark/structure/io/OutputRDD.java | 6 +++++-
.../tinkerpop/gremlin/spark/structure/io/ExampleInputRDD.java | 5 -----
.../gremlin/spark/structure/io/ExampleOutputRDD.java | 6 ------
.../gremlin/spark/structure/io/ToyGraphInputRDD.java | 7 -------
5 files changed, 10 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1d36fea2/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDD.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDD.java
index 982e8ec..c94e46b 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDD.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDD.java
@@ -22,6 +22,7 @@ import org.apache.commons.configuration.Configuration;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import scala.Tuple2;
/**
* An InputRDD is used to read data from the underlying graph system and yield the respective adjacency list.
@@ -42,6 +43,7 @@ public interface InputRDD {
/**
* Read a memoryRDD from the storage location.
+ * The default implementation returns an empty RDD.
*
* @param configuration the configuration for the {@link org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer}
* @param memoryKey the memory key of the memoryRDD
@@ -50,5 +52,7 @@ public interface InputRDD {
* @param <V> the value class of the memoryRDD
* @return the memoryRDD with respective key/value pairs.
*/
- public <K, V> JavaPairRDD<K, V> readMemoryRDD(final Configuration configuration, final String memoryKey, final JavaSparkContext sparkContext);
+ public default <K, V> JavaPairRDD<K, V> readMemoryRDD(final Configuration configuration, final String memoryKey, final JavaSparkContext sparkContext) {
+ return sparkContext.<Tuple2<K, V>>emptyRDD().mapToPair(t -> t);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1d36fea2/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDD.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDD.java
index 6512b83..9c5f6a5 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDD.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDD.java
@@ -23,6 +23,7 @@ import org.apache.spark.api.java.JavaPairRDD;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
import org.apache.tinkerpop.gremlin.process.computer.KeyValue;
+import java.util.Collections;
import java.util.Iterator;
/**
@@ -40,6 +41,7 @@ public interface OutputRDD {
/**
* Write the sideEffect memoryRDD to an output location. The {@link Configuration} maintains the specified location via {@link org.apache.tinkerpop.gremlin.hadoop.Constants#GREMLIN_HADOOP_OUTPUT_LOCATION}.
+ * The default implementation returns an empty iterator.
*
* @param configuration the configuration of the Spark job
* @param memoryKey the memory key of the memoryRDD
@@ -48,5 +50,7 @@ public interface OutputRDD {
* @param <V> the value class of the RDD
* @return the {@link KeyValue} iterator to store in the final resultant {@link org.apache.tinkerpop.gremlin.process.computer.Memory}.
*/
- public <K, V> Iterator<KeyValue<K, V>> writeMemoryRDD(final Configuration configuration, final String memoryKey, final JavaPairRDD<K, V> memoryRDD);
+ public default <K, V> Iterator<KeyValue<K, V>> writeMemoryRDD(final Configuration configuration, final String memoryKey, final JavaPairRDD<K, V> memoryRDD) {
+ return Collections.emptyIterator();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1d36fea2/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ExampleInputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ExampleInputRDD.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ExampleInputRDD.java
index 87c4e2a..86c7610 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ExampleInputRDD.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ExampleInputRDD.java
@@ -48,9 +48,4 @@ public final class ExampleInputRDD implements InputRDD {
list.add(StarGraph.open().addVertex(T.id, 6l, T.label, "person", "age", 35));
return sparkContext.parallelize(list).mapToPair(vertex -> new Tuple2<>(vertex.id(), new VertexWritable(vertex)));
}
-
- @Override
- public <K, V> JavaPairRDD<K, V> readMemoryRDD(Configuration configuration, String memoryKey, JavaSparkContext sparkContext) {
- return null;
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1d36fea2/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ExampleOutputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ExampleOutputRDD.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ExampleOutputRDD.java
index 321adcc..103ec20 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ExampleOutputRDD.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ExampleOutputRDD.java
@@ -21,7 +21,6 @@ package org.apache.tinkerpop.gremlin.spark.structure.io;
import org.apache.commons.configuration.Configuration;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.process.computer.KeyValue;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import java.util.Iterator;
@@ -48,9 +47,4 @@ public final class ExampleOutputRDD implements OutputRDD {
}
assertEquals(123, totalAge);
}
-
- @Override
- public <K, V> Iterator<KeyValue<K, V>> writeMemoryRDD(Configuration configuration, String memoryKey, JavaPairRDD<K, V> memoryRDD) {
- return null;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1d36fea2/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
index 5b9ada9..ea3636f 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
@@ -70,11 +70,4 @@ public final class ToyGraphInputRDD implements InputRDD {
return sparkContext.parallelize(vertices).mapToPair(vertex -> new Tuple2<>(vertex.get().id(), vertex));
}
-
- @Override
- public <K, V> JavaPairRDD<K, V> readMemoryRDD(Configuration configuration, String memoryKey, JavaSparkContext sparkContext) {
- return null;
- }
-
-
}