You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2015/12/03 21:31:14 UTC
[01/17] incubator-tinkerpop git commit: Adjust tests to be a bit more
bulletproof with respect to thread timing.
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/TINKERPOP3-1014 83b93e037 -> 9af660476
Adjust tests to be a bit more bulletproof with respect to thread timing.
Not sure if i've fixed anything with this change as these tests have never failed for me on my system.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/87201658
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/87201658
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/87201658
Branch: refs/heads/TINKERPOP3-1014
Commit: 87201658f2023c725d4620b4a700b949605099a7
Parents: 59eaa06
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Wed Dec 2 14:20:30 2015 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Wed Dec 2 14:20:49 2015 -0500
----------------------------------------------------------------------
.../groovy/engine/GremlinExecutorTest.java | 8 +--
.../jsr223/GremlinGroovyScriptEngineTest.java | 52 ++++++++++++--------
2 files changed, 35 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/87201658/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/GremlinExecutorTest.java
----------------------------------------------------------------------
diff --git a/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/GremlinExecutorTest.java b/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/GremlinExecutorTest.java
index c30ad79..7bc0708 100644
--- a/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/GremlinExecutorTest.java
+++ b/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/GremlinExecutorTest.java
@@ -478,7 +478,7 @@ public class GremlinExecutorTest {
public void shouldContinueToEvalScriptsEvenWithTimedInterrupt() throws Exception {
final Map<String,List<Object>> compilerCustomizerConfig = new HashMap<>();
final List<Object> args = new ArrayList<>();
- args.add(50);
+ args.add(250);
compilerCustomizerConfig.put(TimedInterruptCustomizerProvider.class.getName(), args);
final Map<String, Object> config = new HashMap<>();
@@ -494,14 +494,14 @@ public class GremlinExecutorTest {
for (int ix = 0; ix < 5; ix++) {
try {
- // this script takes 10 ms longer than the interruptionTimeout
- gremlinExecutor.eval("s = System.currentTimeMillis();\nwhile((System.currentTimeMillis() - s) < 100) {}").get();
+ // this script takes significantly longer than the interruptionTimeout
+ gremlinExecutor.eval("s = System.currentTimeMillis();\nwhile((System.currentTimeMillis() - s) < 10000) {}").get();
fail("This should have timed out");
} catch (Exception se) {
assertEquals(TimeoutException.class, se.getCause().getClass());
}
- // this script takes 10 ms less than the interruptionTimeout
+ // this script takes significantly less than the interruptionTimeout
assertEquals("test", gremlinExecutor.eval("s = System.currentTimeMillis();\nwhile((System.currentTimeMillis() - s) < 20) {};'test'").get());
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/87201658/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngineTest.java
----------------------------------------------------------------------
diff --git a/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngineTest.java b/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngineTest.java
index 9e74686..f3b3835 100644
--- a/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngineTest.java
+++ b/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngineTest.java
@@ -29,12 +29,17 @@ import javax.script.ScriptContext;
import javax.script.ScriptEngine;
import javax.script.ScriptException;
import javax.script.SimpleBindings;
+import java.awt.*;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.greaterThan;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -203,51 +208,56 @@ public class GremlinGroovyScriptEngineTest {
@Test
public void shouldReloadClassLoaderWhileDoingEvalInSeparateThread() throws Exception {
final AtomicBoolean fail = new AtomicBoolean(false);
+ final AtomicInteger counter = new AtomicInteger(0);
final CountDownLatch latch = new CountDownLatch(1);
+ final AtomicReference<Color> color = new AtomicReference<>(Color.RED);
+
final GremlinGroovyScriptEngine scriptEngine = new GremlinGroovyScriptEngine();
- final Thread t = new Thread(() -> {
- try {
- final Object o = scriptEngine.eval("Color.BLACK");
- System.out.println("Should not print: " + o);
- fail.set(true);
- } catch (ScriptException se) {
- // should get here as Color.BLACK is not imported yet.
- System.out.println("Failed to execute Color.BLACK as expected.");
- }
+ try {
+ scriptEngine.eval("Color.BLACK");
+ fail("Should fail as class is not yet imported");
+ } catch (ScriptException se) {
+ // should get here as Color.BLACK is not imported yet.
+ System.out.println("Failed to execute Color.BLACK as expected.");
+ }
+
+ final Thread evalThread = new Thread(() -> {
try {
- int counter = 0;
+ // execute scripts until the other thread releases this latch (i.e. after import)
while (latch.getCount() == 1) {
scriptEngine.eval("1+1");
- counter++;
+ counter.incrementAndGet();
}
- System.out.println(counter + " executions.");
-
- scriptEngine.eval("Color.BLACK");
- System.out.println("Color.BLACK now evaluates");
+ color.set((Color) scriptEngine.eval("Color.BLACK"));
} catch (Exception se) {
- se.printStackTrace();
fail.set(true);
}
}, "test-reload-classloader-1");
- t.start();
+ evalThread.start();
- // let the first thead execute a bit.
+ // let the first thread execute a bit.
Thread.sleep(1000);
- new Thread(() -> {
+ final Thread importThread = new Thread(() -> {
System.out.println("Importing java.awt.Color...");
final Set<String> imports = new HashSet<String>() {{
add("import java.awt.Color");
}};
scriptEngine.addImports(imports);
latch.countDown();
- }, "test-reload-classloader-2").start();
+ }, "test-reload-classloader-2");
+
+ importThread.start();
- t.join();
+ // block until both threads are done
+ importThread.join();
+ evalThread.join();
+ assertEquals(Color.BLACK, color.get());
+ assertThat(counter.get(), greaterThan(0));
assertFalse(fail.get());
}
[05/17] incubator-tinkerpop git commit: HadoopElementIterator now
supports ANY InputFormat, not just FileInputFormat. Sweet. Also,
if you are using an RDD in Spark (and thus,
not really doing Hadoop InputFormat stuffs),
we have InputRDDFormat which wraps
Posted by ok...@apache.org.
HadoopElementIterator now supports ANY InputFormat, not just FileInputFormat. Sweet. Also, if you are using an RDD in Spark (and thus, not really doing Hadoop InputFormat stuffs), we have InputRDDFormat which wraps an RDD in an InputFormat so HadoopElementIterator works as well. This solves the HadoopGraph OLTP problem for ALL InputFormats and it allows ComputerResultStep to Attach elements for more than just FileInputFormats. Good stuff.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/85eb63db
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/85eb63db
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/85eb63db
Branch: refs/heads/TINKERPOP3-1014
Commit: 85eb63dbc4855f7f3e944b82af8c9aae03c8aed2
Parents: 5b9de83
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Dec 1 09:25:14 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Dec 3 12:47:52 2015 -0700
----------------------------------------------------------------------
.../structure/hdfs/HadoopEdgeIterator.java | 4 -
.../structure/hdfs/HadoopElementIterator.java | 35 +++--
.../structure/hdfs/HadoopVertexIterator.java | 4 -
.../HadoopGraphStructureStandardTest.java | 35 +++++
.../io/HadoopGraphStructureStandardTest.java | 35 -----
.../spark/structure/io/InputRDDFormat.java | 136 +++++++++++++++++++
.../spark/structure/io/InputRDDTest.java | 21 +++
7 files changed, 208 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/85eb63db/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopEdgeIterator.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopEdgeIterator.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopEdgeIterator.java
index acdc0a4..59a4d2c 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopEdgeIterator.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopEdgeIterator.java
@@ -39,10 +39,6 @@ public final class HadoopEdgeIterator extends HadoopElementIterator<Edge> {
private Iterator<Edge> edgeIterator = Collections.emptyIterator();
- public HadoopEdgeIterator(final HadoopGraph graph, final InputFormat<NullWritable, VertexWritable> inputFormat, final Path path) throws IOException, InterruptedException {
- super(graph, inputFormat, path);
- }
-
public HadoopEdgeIterator(final HadoopGraph graph) throws IOException {
super(graph);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/85eb63db/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopElementIterator.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopElementIterator.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopElementIterator.java
index a9f287b..f9ffea2 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopElementIterator.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopElementIterator.java
@@ -19,13 +19,14 @@
package org.apache.tinkerpop.gremlin.hadoop.structure.hdfs;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptID;
-import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.task.JobContextImpl;
import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
@@ -36,35 +37,31 @@ import org.apache.tinkerpop.gremlin.structure.Element;
import java.io.IOException;
import java.util.Iterator;
import java.util.LinkedList;
+import java.util.List;
import java.util.Queue;
+import java.util.UUID;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
public abstract class HadoopElementIterator<E extends Element> implements Iterator<E> {
- // TODO: Generalize so it works for more than just FileFormats.
-
protected final HadoopGraph graph;
protected final Queue<RecordReader<NullWritable, VertexWritable>> readers = new LinkedList<>();
- public HadoopElementIterator(final HadoopGraph graph, final InputFormat<NullWritable, VertexWritable> inputFormat, final Path path) throws IOException, InterruptedException {
- this.graph = graph;
- final Configuration configuration = ConfUtil.makeHadoopConfiguration(this.graph.configuration());
- for (final Path path2 : HDFSTools.getAllFilePaths(FileSystem.get(configuration), path, HiddenFileFilter.instance())) {
- this.readers.add(inputFormat.createRecordReader(new FileSplit(path2, 0, Long.MAX_VALUE, new String[]{}), new TaskAttemptContextImpl(configuration, new TaskAttemptID())));
- }
- }
-
public HadoopElementIterator(final HadoopGraph graph) throws IOException {
try {
this.graph = graph;
- if (this.graph.configuration().containsKey(Constants.GREMLIN_HADOOP_INPUT_LOCATION)) {
- final Configuration configuration = ConfUtil.makeHadoopConfiguration(this.graph.configuration());
- final InputFormat<NullWritable, VertexWritable> inputFormat = this.graph.configuration().getGraphInputFormat().getConstructor().newInstance();
- for (final Path path : HDFSTools.getAllFilePaths(FileSystem.get(configuration), new Path(graph.configuration().getInputLocation()), HiddenFileFilter.instance())) {
- this.readers.add(inputFormat.createRecordReader(new FileSplit(path, 0, Long.MAX_VALUE, new String[]{}), new TaskAttemptContextImpl(configuration, new TaskAttemptID())));
- }
+ final Configuration configuration = ConfUtil.makeHadoopConfiguration(this.graph.configuration());
+ final InputFormat<NullWritable, VertexWritable> inputFormat = this.graph.configuration().getGraphInputFormat().getConstructor().newInstance();
+ if (inputFormat instanceof FileInputFormat) {
+ if (!this.graph.configuration().containsKey(Constants.GREMLIN_HADOOP_INPUT_LOCATION))
+ return; // there is not input location and thus, no data (empty graph)
+ configuration.set(Constants.MAPREDUCE_INPUT_FILEINPUTFORMAT_INPUTDIR, this.graph.configuration().getInputLocation());
+ }
+ final List<InputSplit> splits = inputFormat.getSplits(new JobContextImpl(configuration, new JobID(UUID.randomUUID().toString(), 1)));
+ for (final InputSplit split : splits) {
+ this.readers.add(inputFormat.createRecordReader(split, new TaskAttemptContextImpl(configuration, new TaskAttemptID())));
}
} catch (Exception e) {
throw new IllegalStateException(e.getMessage(), e);
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/85eb63db/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopVertexIterator.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopVertexIterator.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopVertexIterator.java
index 8977692..8f13c59 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopVertexIterator.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopVertexIterator.java
@@ -36,10 +36,6 @@ public final class HadoopVertexIterator extends HadoopElementIterator<Vertex> {
private HadoopVertex nextVertex = null;
- public HadoopVertexIterator(final HadoopGraph graph, final InputFormat<NullWritable, VertexWritable> inputFormat, final Path path) throws IOException, InterruptedException {
- super(graph, inputFormat, path);
- }
-
public HadoopVertexIterator(final HadoopGraph graph) throws IOException {
super(graph);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/85eb63db/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraphStructureStandardTest.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraphStructureStandardTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraphStructureStandardTest.java
new file mode 100644
index 0000000..459c900
--- /dev/null
+++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraphStructureStandardTest.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.hadoop.structure;
+
+import org.apache.tinkerpop.gremlin.GraphProviderClass;
+import org.apache.tinkerpop.gremlin.hadoop.HadoopGraphProvider;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.structure.StructureStandardSuite;
+import org.junit.runner.RunWith;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+@RunWith(StructureStandardSuite.class)
+@GraphProviderClass(provider = HadoopGraphProvider.class, graph = HadoopGraph.class)
+public class HadoopGraphStructureStandardTest {
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/85eb63db/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopGraphStructureStandardTest.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopGraphStructureStandardTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopGraphStructureStandardTest.java
deleted file mode 100644
index ccb2d64..0000000
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopGraphStructureStandardTest.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.structure.io;
-
-import org.apache.tinkerpop.gremlin.GraphProviderClass;
-import org.apache.tinkerpop.gremlin.hadoop.HadoopGraphProvider;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.structure.StructureStandardSuite;
-import org.junit.runner.RunWith;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-@RunWith(StructureStandardSuite.class)
-@GraphProviderClass(provider = HadoopGraphProvider.class, graph = HadoopGraph.class)
-public class HadoopGraphStructureStandardTest {
-}
-
-
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/85eb63db/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDFormat.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDFormat.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDFormat.java
new file mode 100644
index 0000000..3952c66
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDFormat.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.structure.io;
+
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.spark.SparkConf;
+import org.apache.spark.SparkContext;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
+import scala.Tuple2;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.UUID;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class InputRDDFormat extends InputFormat<NullWritable, VertexWritable> {
+
+ public InputRDDFormat() {
+
+ }
+
+ @Override
+ public List<InputSplit> getSplits(final JobContext jobContext) throws IOException, InterruptedException {
+ return Collections.singletonList(new InputSplit() {
+ @Override
+ public long getLength() throws IOException, InterruptedException {
+ return 0;
+ }
+
+ @Override
+ public String[] getLocations() throws IOException, InterruptedException {
+ return new String[0];
+ }
+ });
+ }
+
+ @Override
+ public RecordReader<NullWritable, VertexWritable> createRecordReader(final InputSplit inputSplit, final TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
+ try {
+ final org.apache.hadoop.conf.Configuration hadoopConfiguration = taskAttemptContext.getConfiguration();
+ final SparkConf sparkConfiguration = new SparkConf();
+ sparkConfiguration.setAppName(UUID.randomUUID().toString());
+ hadoopConfiguration.forEach(entry -> sparkConfiguration.set(entry.getKey(), entry.getValue()));
+ InputRDD inputRDD = (InputRDD) Class.forName(sparkConfiguration.get(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD)).newInstance();
+ JavaSparkContext javaSparkContext = new JavaSparkContext(SparkContext.getOrCreate(sparkConfiguration));
+ final Iterator<Tuple2<Object, VertexWritable>> iterator = inputRDD.readGraphRDD(ConfUtil.makeApacheConfiguration(taskAttemptContext.getConfiguration()), javaSparkContext).toLocalIterator();
+ return new RecordReader<NullWritable, VertexWritable>() {
+ @Override
+ public void initialize(final InputSplit inputSplit, final TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
+
+ }
+
+ @Override
+ public boolean nextKeyValue() throws IOException, InterruptedException {
+ return iterator.hasNext();
+ }
+
+ @Override
+ public NullWritable getCurrentKey() throws IOException, InterruptedException {
+ return NullWritable.get();
+ }
+
+ @Override
+ public VertexWritable getCurrentValue() throws IOException, InterruptedException {
+ return iterator.next()._2();
+ }
+
+ @Override
+ public float getProgress() throws IOException, InterruptedException {
+ return 1.0f; // TODO: make this dynamic (how? its an iterator.)
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (!hadoopConfiguration.getBoolean(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, false))
+ javaSparkContext.close();
+ }
+ };
+ } catch (final ClassNotFoundException | InstantiationException | IllegalAccessException e) {
+ throw new IOException(e.getMessage(), e);
+ }
+
+ }
+
+ /*private static class PartitionInputSplit extends InputSplit {
+
+ private final Partition partition;
+
+ public PartitionInputSplit(final Partition partition) {
+ this.partition = partition;
+ }
+
+ @Override
+ public long getLength() throws IOException, InterruptedException {
+ return 0;
+ }
+
+ @Override
+ public String[] getLocations() throws IOException, InterruptedException {
+ return new String[0];
+ }
+
+ public Partition getPartition() {
+ return this.partition;
+ }
+ }*/
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/85eb63db/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
index 5ba3b12..98a2b9f 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
@@ -24,6 +24,7 @@ import org.apache.spark.serializer.KryoSerializer;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
+import org.apache.tinkerpop.gremlin.process.traversal.P;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
import org.apache.tinkerpop.gremlin.structure.Graph;
@@ -52,4 +53,24 @@ public class InputRDDTest {
assertEquals(123l, graph.traversal(GraphTraversalSource.computer(SparkGraphComputer.class)).V().values("age").sum().next());
assertEquals(Long.valueOf(4l), graph.traversal(GraphTraversalSource.computer(SparkGraphComputer.class)).V().count().next());
}
+
+ @Test
+ public void shouldSupportHadoopGraphOLTP() {
+ final Configuration configuration = new BaseConfiguration();
+ configuration.setProperty("spark.master", "local[4]");
+ configuration.setProperty("spark.serializer", KryoSerializer.class.getCanonicalName());
+ configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
+ configuration.setProperty(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, ExampleInputRDD.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT, GryoOutputFormat.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, "target/test-output");
+ configuration.setProperty(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
+ ////////
+ Graph graph = GraphFactory.open(configuration);
+ GraphTraversalSource g = graph.traversal(); // OLTP;
+ assertEquals("person", g.V().has("age", 29).next().label());
+ assertEquals(Long.valueOf(4), g.V().count().next());
+ assertEquals(Long.valueOf(0), g.E().count().next());
+ assertEquals(Long.valueOf(2), g.V().has("age", P.gt(30)).count().next());
+ }
}
[10/17] incubator-tinkerpop git commit: updated CHANGELOG.
Posted by ok...@apache.org.
updated CHANGELOG.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/461be033
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/461be033
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/461be033
Branch: refs/heads/TINKERPOP3-1014
Commit: 461be033b6b11ef97c0ae7af0b88d1beb1a326f9
Parents: 87382a9
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Thu Dec 3 07:49:40 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Dec 3 12:47:53 2015 -0700
----------------------------------------------------------------------
CHANGELOG.asciidoc | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/461be033/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 9f45a90..90c9f27 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,6 +26,10 @@ image::https://raw.githubusercontent.com/apache/incubator-tinkerpop/master/docs/
TinkerPop 3.1.1 (NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* Added `AbstractSparkTest` which handles closing `SparkContext` instances between tests now that we support persisted contexts.
+* Fixed a serialization bug in `GryoSerializer` that made it difficult for graph providers to yield `InputRDDs` for `SparkGraphComputer`.
+* `SparkGraphComputer` is now tested against Gryo, GraphSON, and `InputRDD` data sources.
+* `HadoopElementIterator` (for Hadoop-Gremlin OLTP) now works for any `InputFormat`, not just `FileInputFormats`.
* Added `Traverser.Admin.getTags()` which are used to mark branches in a traversal (useful in `match()` and related future steps).
* Fixed the `Future` model for `GiraphGraphComputer` and `SparkGraphComputer` so that class loaders are preserved.
* Added support for arbitrary vertex ID types in `BulkLoaderVertexProgram`.
[08/17] incubator-tinkerpop git commit: I have the
SparkIntegrationTestSuite now testing either from Gryo FileInputFormat,
GraphSON FileInputFormat,
or an InputRDD. This gives us super coverage and proves that InputRDD
(bypassing Hadoop) is working as ex
Posted by ok...@apache.org.
I have the SparkIntegrationTestSuite now testing either from Gryo FileInputFormat, GraphSON FileInputFormat, or an InputRDD. This gives us super coverage and proves that InputRDD (bypassing Hadoop) is working as expected. I also fixed up some other tests that used KryoSerializer instead of GryoSerializer as I learned how to deal with Scalas WrappedArray class. It was insane. This is really good stuff.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/ddbbec08
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/ddbbec08
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/ddbbec08
Branch: refs/heads/TINKERPOP3-1014
Commit: ddbbec084635a7e764632713a07b0c532a614e4c
Parents: 85eb63d
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed Dec 2 12:02:09 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Dec 3 12:47:52 2015 -0700
----------------------------------------------------------------------
.../structure/hdfs/HadoopElementIterator.java | 6 ++-
.../computer/payload/MessagePayload.java | 5 +-
.../computer/payload/ViewIncomingPayload.java | 6 +--
.../computer/payload/ViewOutgoingPayload.java | 12 +++--
.../spark/structure/io/gryo/GryoSerializer.java | 6 +++
.../io/gryo/WrappedArraySerializer.java | 46 ++++++++++++++++++
.../computer/SparkHadoopGraphProvider.java | 40 ++++++++++++++--
.../spark/structure/io/ClassicInputRDD.java | 39 +++++++++++++++
.../spark/structure/io/GratefulInputRDD.java | 50 ++++++++++++++++++++
.../spark/structure/io/InputOutputRDDTest.java | 3 +-
.../spark/structure/io/InputRDDTest.java | 5 +-
.../spark/structure/io/ModernInputRDD.java | 41 ++++++++++++++++
.../spark/structure/io/OutputRDDTest.java | 3 +-
.../spark/structure/io/TheCrewInputRDD.java | 39 +++++++++++++++
14 files changed, 285 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ddbbec08/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopElementIterator.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopElementIterator.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopElementIterator.java
index f9ffea2..45f3c55 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopElementIterator.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopElementIterator.java
@@ -19,6 +19,8 @@
package org.apache.tinkerpop.gremlin.hadoop.structure.hdfs;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.InputSplit;
@@ -56,7 +58,9 @@ public abstract class HadoopElementIterator<E extends Element> implements Iterat
final InputFormat<NullWritable, VertexWritable> inputFormat = this.graph.configuration().getGraphInputFormat().getConstructor().newInstance();
if (inputFormat instanceof FileInputFormat) {
if (!this.graph.configuration().containsKey(Constants.GREMLIN_HADOOP_INPUT_LOCATION))
- return; // there is not input location and thus, no data (empty graph)
+ return; // there is no input location and thus, no data (empty graph)
+ if (!FileSystem.get(configuration).exists(new Path(this.graph.configuration().getInputLocation())))
+ return; // there is no data at the input location (empty graph)
configuration.set(Constants.MAPREDUCE_INPUT_FILEINPUTFORMAT_INPUTDIR, this.graph.configuration().getInputLocation());
}
final List<InputSplit> splits = inputFormat.getSplits(new JobContextImpl(configuration, new JobID(UUID.randomUUID().toString(), 1)));
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ddbbec08/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/MessagePayload.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/MessagePayload.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/MessagePayload.java
index 09e2599..f32ec44 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/MessagePayload.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/MessagePayload.java
@@ -23,7 +23,10 @@ package org.apache.tinkerpop.gremlin.spark.process.computer.payload;
*/
public final class MessagePayload<M> implements Payload {
- private final M message;
+ private M message;
+
+ private MessagePayload() {
+ }
public MessagePayload(final M message) {
this.message = message;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ddbbec08/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewIncomingPayload.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewIncomingPayload.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewIncomingPayload.java
index 911fc7b..a2c9205 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewIncomingPayload.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewIncomingPayload.java
@@ -31,11 +31,11 @@ import java.util.List;
public final class ViewIncomingPayload<M> implements Payload {
private List<DetachedVertexProperty<Object>> view = null;
- private final List<M> incomingMessages;
+ private List<M> incomingMessages;
- public ViewIncomingPayload() {
- this.incomingMessages = null;
+ private ViewIncomingPayload() {
+
}
public ViewIncomingPayload(final MessageCombiner<M> messageCombiner) {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ddbbec08/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewOutgoingPayload.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewOutgoingPayload.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewOutgoingPayload.java
index fc4aeed..20c8e09 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewOutgoingPayload.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewOutgoingPayload.java
@@ -28,10 +28,14 @@ import java.util.List;
*/
public final class ViewOutgoingPayload<M> implements Payload {
- private final List<DetachedVertexProperty<Object>> view;
- private final List<Tuple2<Object,M>> outgoingMessages;
+ private List<DetachedVertexProperty<Object>> view;
+ private List<Tuple2<Object, M>> outgoingMessages;
- public ViewOutgoingPayload(final List<DetachedVertexProperty<Object>> view, final List<Tuple2<Object,M>> outgoingMessages) {
+ private ViewOutgoingPayload() {
+
+ }
+
+ public ViewOutgoingPayload(final List<DetachedVertexProperty<Object>> view, final List<Tuple2<Object, M>> outgoingMessages) {
this.view = view;
this.outgoingMessages = outgoingMessages;
}
@@ -40,7 +44,7 @@ public final class ViewOutgoingPayload<M> implements Payload {
return new ViewPayload(this.view);
}
- public List<Tuple2<Object,M>> getOutgoingMessages() {
+ public List<Tuple2<Object, M>> getOutgoingMessages() {
return this.outgoingMessages;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ddbbec08/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java
index 7202892..29fde9f 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java
@@ -41,6 +41,8 @@ import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoPool;
import org.apache.tinkerpop.shaded.kryo.io.Output;
import org.apache.tinkerpop.shaded.kryo.serializers.JavaSerializer;
import scala.Tuple2;
+import scala.Tuple3;
+import scala.collection.mutable.WrappedArray;
import scala.runtime.BoxedUnit;
import java.util.Collections;
@@ -79,11 +81,15 @@ public final class GryoSerializer extends Serializer {
try {
builder.addCustom(SerializableWritable.class, new JavaSerializer())
.addCustom(Tuple2.class, new JavaSerializer())
+ .addCustom(Tuple2[].class, new JavaSerializer())
+ .addCustom(Tuple3.class, new JavaSerializer())
+ .addCustom(Tuple3[].class, new JavaSerializer())
.addCustom(CompressedMapStatus.class, new JavaSerializer())
.addCustom(HttpBroadcast.class, new JavaSerializer())
.addCustom(PythonBroadcast.class, new JavaSerializer())
.addCustom(BoxedUnit.class, new JavaSerializer())
.addCustom(Class.forName("scala.reflect.ClassTag$$anon$1"), new JavaSerializer())
+ .addCustom(WrappedArray.ofRef.class, new WrappedArraySerializer())
.addCustom(MessagePayload.class)
.addCustom(ViewIncomingPayload.class)
.addCustom(ViewOutgoingPayload.class)
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ddbbec08/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/WrappedArraySerializer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/WrappedArraySerializer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/WrappedArraySerializer.java
new file mode 100644
index 0000000..0e9f03f
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/WrappedArraySerializer.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.structure.io.gryo;
+
+import org.apache.tinkerpop.shaded.kryo.Kryo;
+import org.apache.tinkerpop.shaded.kryo.Serializer;
+import org.apache.tinkerpop.shaded.kryo.io.Input;
+import org.apache.tinkerpop.shaded.kryo.io.Output;
+import scala.collection.JavaConversions;
+import scala.collection.mutable.WrappedArray;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class WrappedArraySerializer<T> extends Serializer<WrappedArray<T>> {
+
+ @Override
+ public void write(final Kryo kryo, final Output output, final WrappedArray<T> iterable) {
+ kryo.writeClassAndObject(output,new ArrayList<>(JavaConversions.asJavaList(iterable)));
+ }
+
+ @Override
+ public WrappedArray<T> read(final Kryo kryo, final Input input, final Class<WrappedArray<T>> aClass) {
+ return new WrappedArray.ofRef<>((T[]) ((List<T>) kryo.readClassAndObject(input)).toArray());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ddbbec08/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
index c81ea92..2328176 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
@@ -20,12 +20,19 @@ package org.apache.tinkerpop.gremlin.spark.process.computer;
import org.apache.tinkerpop.gremlin.GraphProvider;
import org.apache.tinkerpop.gremlin.LoadGraphWith;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.HadoopGraphProvider;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
+import org.apache.tinkerpop.gremlin.spark.structure.io.ClassicInputRDD;
+import org.apache.tinkerpop.gremlin.spark.structure.io.GratefulInputRDD;
+import org.apache.tinkerpop.gremlin.spark.structure.io.InputRDDFormat;
+import org.apache.tinkerpop.gremlin.spark.structure.io.ModernInputRDD;
+import org.apache.tinkerpop.gremlin.spark.structure.io.TheCrewInputRDD;
import org.apache.tinkerpop.gremlin.structure.Graph;
import java.util.Map;
+import java.util.Random;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
@@ -33,13 +40,41 @@ import java.util.Map;
@GraphProvider.Descriptor(computer = SparkGraphComputer.class)
public final class SparkHadoopGraphProvider extends HadoopGraphProvider {
+ private static final Random RANDOM = new Random();
+
@Override
public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
final Map<String, Object> config = super.getBaseConfiguration(graphName, test, testMethodName, loadGraphWith);
- config.put("mapreduce.job.reduces", 4);
+ if (null != loadGraphWith) {
+ if (loadGraphWith.equals(LoadGraphWith.GraphData.MODERN)) {
+ if (RANDOM.nextBoolean()) {
+ config.remove(Constants.GREMLIN_HADOOP_INPUT_LOCATION);
+ config.put(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, ModernInputRDD.class.getCanonicalName());
+ config.put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName());
+ }
+ } else if (loadGraphWith.equals(LoadGraphWith.GraphData.CREW)) {
+ if (RANDOM.nextBoolean()) {
+ config.remove(Constants.GREMLIN_HADOOP_INPUT_LOCATION);
+ config.put(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, TheCrewInputRDD.class.getCanonicalName());
+ config.put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName());
+ }
+ } else if (loadGraphWith.equals(LoadGraphWith.GraphData.CLASSIC)) {
+ if (RANDOM.nextBoolean()) {
+ config.remove(Constants.GREMLIN_HADOOP_INPUT_LOCATION);
+ config.put(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, ClassicInputRDD.class.getCanonicalName());
+ config.put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName());
+ }
+ } else if (loadGraphWith.equals(LoadGraphWith.GraphData.GRATEFUL)) {
+ if (RANDOM.nextBoolean()) {
+ config.remove(Constants.GREMLIN_HADOOP_INPUT_LOCATION);
+ config.put(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, GratefulInputRDD.class.getCanonicalName());
+ config.put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName());
+ }
+ }
+ }
/// spark configuration
config.put("spark.master", "local[4]");
- // put("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
+ //config.put("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
config.put("spark.serializer", "org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer");
config.put("spark.kryo.registrationRequired", true);
return config;
@@ -49,5 +84,4 @@ public final class SparkHadoopGraphProvider extends HadoopGraphProvider {
public GraphTraversalSource traversal(final Graph graph) {
return GraphTraversalSource.build().engine(ComputerTraversalEngine.build().computer(SparkGraphComputer.class)).create(graph);
}
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ddbbec08/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ClassicInputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ClassicInputRDD.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ClassicInputRDD.java
new file mode 100644
index 0000000..4512b61
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ClassicInputRDD.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.structure.io;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+import scala.Tuple2;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class ClassicInputRDD implements InputRDD {
+
+ @Override
+ public JavaPairRDD<Object, VertexWritable> readGraphRDD(final Configuration configuration, final JavaSparkContext sparkContext) {
+ return sparkContext.parallelize(IteratorUtils.list(IteratorUtils.map(TinkerFactory.createClassic().vertices(), VertexWritable::new))).mapToPair(vertex -> new Tuple2<>(vertex.get().id(), vertex));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ddbbec08/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/GratefulInputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/GratefulInputRDD.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/GratefulInputRDD.java
new file mode 100644
index 0000000..396aa75
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/GratefulInputRDD.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.structure.io;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+import scala.Tuple2;
+
+import java.io.IOException;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class GratefulInputRDD implements InputRDD {
+
+ @Override
+ public JavaPairRDD<Object, VertexWritable> readGraphRDD(final Configuration configuration, final JavaSparkContext sparkContext) {
+ try {
+ final Graph graph = TinkerGraph.open();
+ graph.io(GryoIo.build()).readGraph(GryoResourceAccess.class.getResource("grateful-dead.kryo").getFile());
+ return sparkContext.parallelize(IteratorUtils.list(IteratorUtils.map(graph.vertices(), VertexWritable::new))).mapToPair(vertex -> new Tuple2<>(vertex.get().id(), vertex));
+ } catch (final IOException e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ddbbec08/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputRDDTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputRDDTest.java
index 3691aba..ea62114 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputRDDTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputRDDTest.java
@@ -28,6 +28,7 @@ import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexPr
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
+import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
import org.junit.Test;
@@ -41,7 +42,7 @@ public class InputOutputRDDTest {
public void shouldReadFromWriteToArbitraryRDD() throws Exception {
final Configuration configuration = new BaseConfiguration();
configuration.setProperty("spark.master", "local[4]");
- configuration.setProperty("spark.serializer", KryoSerializer.class.getCanonicalName());
+ configuration.setProperty("spark.serializer", GryoSerializer.class.getCanonicalName());
configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
configuration.setProperty(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, ExampleInputRDD.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_SPARK_GRAPH_OUTPUT_RDD, ExampleOutputRDD.class.getCanonicalName());
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ddbbec08/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
index 98a2b9f..2cbfd66 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
@@ -27,6 +27,7 @@ import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
import org.apache.tinkerpop.gremlin.process.traversal.P;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
+import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
import org.junit.Test;
@@ -42,7 +43,7 @@ public class InputRDDTest {
public void shouldReadFromArbitraryRDD() {
final Configuration configuration = new BaseConfiguration();
configuration.setProperty("spark.master", "local[4]");
- configuration.setProperty("spark.serializer", KryoSerializer.class.getCanonicalName());
+ configuration.setProperty("spark.serializer", GryoSerializer.class.getCanonicalName());
configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
configuration.setProperty(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, ExampleInputRDD.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT, GryoOutputFormat.class.getCanonicalName());
@@ -58,7 +59,7 @@ public class InputRDDTest {
public void shouldSupportHadoopGraphOLTP() {
final Configuration configuration = new BaseConfiguration();
configuration.setProperty("spark.master", "local[4]");
- configuration.setProperty("spark.serializer", KryoSerializer.class.getCanonicalName());
+ configuration.setProperty("spark.serializer", GryoSerializer.class.getCanonicalName());
configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
configuration.setProperty(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, ExampleInputRDD.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName());
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ddbbec08/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ModernInputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ModernInputRDD.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ModernInputRDD.java
new file mode 100644
index 0000000..849e3e6
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ModernInputRDD.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.structure.io;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+import scala.Tuple2;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class ModernInputRDD implements InputRDD {
+
+ @Override
+ public JavaPairRDD<Object, VertexWritable> readGraphRDD(final Configuration configuration, final JavaSparkContext sparkContext) {
+ return sparkContext.
+ parallelize(IteratorUtils.list(IteratorUtils.map(TinkerFactory.createModern().vertices(), VertexWritable::new))).
+ mapToPair(vertex -> new Tuple2<>(vertex.get().id(), vertex));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ddbbec08/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java
index 10eecb3..f9b6f39 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java
@@ -30,6 +30,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSo
import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkHadoopGraphProvider;
+import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
import org.junit.Test;
@@ -43,7 +44,7 @@ public class OutputRDDTest {
public void shouldWriteToArbitraryRDD() throws Exception {
final Configuration configuration = new BaseConfiguration();
configuration.setProperty("spark.master", "local[4]");
- configuration.setProperty("spark.serializer", KryoSerializer.class.getCanonicalName());
+ configuration.setProperty("spark.serializer", GryoSerializer.class.getCanonicalName());
configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, GryoInputFormat.class.getCanonicalName());
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ddbbec08/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/TheCrewInputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/TheCrewInputRDD.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/TheCrewInputRDD.java
new file mode 100644
index 0000000..ff5a274
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/TheCrewInputRDD.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.structure.io;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+import scala.Tuple2;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class TheCrewInputRDD implements InputRDD {
+
+ @Override
+ public JavaPairRDD<Object, VertexWritable> readGraphRDD(final Configuration configuration, final JavaSparkContext sparkContext) {
+ return sparkContext.parallelize(IteratorUtils.list(IteratorUtils.map(TinkerFactory.createTheCrew().vertices(), VertexWritable::new))).mapToPair(vertex -> new Tuple2<>(vertex.get().id(), vertex));
+ }
+}
[11/17] incubator-tinkerpop git commit: added more docs to Order Step
around Columm.keys and Column.values. Added a GraphTraversal.by() safeguard
to ensure no ambiguous method call issues.
Posted by ok...@apache.org.
added more docs to Order Step around Columm.keys and Column.values. Added a GraphTraversal.by() safeguard to ensure no ambiguous method call issues.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/4795042e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/4795042e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/4795042e
Branch: refs/heads/TINKERPOP3-1014
Commit: 4795042eab7f047cbde57237b9900c43054e0c31
Parents: 447d705
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Thu Dec 3 09:42:27 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Dec 3 12:47:53 2015 -0700
----------------------------------------------------------------------
docs/src/reference/the-traversal.asciidoc | 17 +++++++++++++-
.../gremlin/process/traversal/Order.java | 24 ++++++++++----------
.../traversal/dsl/graph/GraphTraversal.java | 8 ++++---
.../process/traversal/step/map/OrderTest.java | 2 +-
4 files changed, 34 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4795042e/docs/src/reference/the-traversal.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/the-traversal.asciidoc b/docs/src/reference/the-traversal.asciidoc
index 6ce4541..271438b 100644
--- a/docs/src/reference/the-traversal.asciidoc
+++ b/docs/src/reference/the-traversal.asciidoc
@@ -1197,9 +1197,24 @@ g.V().hasLabel('person').order().by(shuffle)
g.V().hasLabel('person').order().by(shuffle)
----
-IMPORTANT: `order(local)` orders the current, local object (not the objects in the traversal stream). This works for
+It is possible to use `order(local)` to order the current local object and not the entire traversal stream. This works for
`Collection`- and `Map`-type objects. For any other object, the object is returned unchanged.
+[gremlin-groovy,modern]
+----
+g.V().values('age').fold().order(local).by(decr) <1>
+g.V().values('age').order(local).by(decr) <2>
+g.V().groupCount().by(inE().count()).order(local).by(values, decr) <3>
+g.V().groupCount().by(inE().count()).order(local).by(keys, incr) <4>
+----
+
+<1> The ages are gathered into a list and then that list is sorted in decreasing order.
+<2> The ages are not gathered and thus `order(local)` is "ordering" single integers and thus, does nothing.
+<3> The `groupCount()` map is ordered by its values in decreasing order.
+<4> The `groupCount()` map is ordered by its keys in increasing order.
+
+NOTE: The `values` and `keys` enums are from `Column` which is used to select "columns" from a `Map`, `Map.Entry`, or `Path`.
+
[[path-step]]
Path Step
~~~~~~~~~
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4795042e/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/Order.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/Order.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/Order.java
index 4e3801a..72fe715 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/Order.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/Order.java
@@ -48,10 +48,10 @@ public enum Order implements Comparator<Object> {
return incr;
}
},
- /**
+ /*
* @deprecated Use {@link org.apache.tinkerpop.gremlin.structure.Column#keys};
- */
- @Deprecated
+ @Deprecated
+ */
keyIncr {
@Override
public int compare(final Object first, final Object second) {
@@ -63,10 +63,10 @@ public enum Order implements Comparator<Object> {
return keyDecr;
}
},
- /**
+ /*
* @deprecated Use {@link org.apache.tinkerpop.gremlin.structure.Column#values};
- */
- @Deprecated
+ *
+ @Deprecated */
valueIncr {
@Override
public int compare(final Object first, final Object second) {
@@ -78,10 +78,10 @@ public enum Order implements Comparator<Object> {
return valueDecr;
}
},
- /**
+ /*
* @deprecated Use {@link org.apache.tinkerpop.gremlin.structure.Column#keys};
- */
- @Deprecated
+ *
+ @Deprecated */
keyDecr {
@Override
public int compare(final Object first, final Object second) {
@@ -93,10 +93,10 @@ public enum Order implements Comparator<Object> {
return keyIncr;
}
},
- /**
+ /*
* @deprecated Use {@link org.apache.tinkerpop.gremlin.structure.Column#values};
- */
- @Deprecated
+ *
+ @Deprecated */
valueDecr {
@Override
public int compare(final Object first, final Object second) {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4795042e/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java
index 3673bce..5dd963c 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java
@@ -1164,7 +1164,7 @@ public interface GraphTraversal<S, E> extends Traversal<S, E> {
}
- ////
+ //// PROJECTION BY-MODULATORS
public default GraphTraversal<S, E> by(final Traversal<?, ?> byTraversal) {
((TraversalParent) this.asAdmin().getEndStep()).addLocalChild(byTraversal.asAdmin());
@@ -1187,7 +1187,7 @@ public interface GraphTraversal<S, E> extends Traversal<S, E> {
return this.by(new ElementValueTraversal<>(elementPropertyKey));
}
- ////
+ //// COMPARATOR BY-MODULATORS
public default GraphTraversal<S, E> by(final Comparator<E> comparator) {
((ComparatorHolder<E>) this.asAdmin().getEndStep()).addComparator(comparator);
@@ -1204,7 +1204,9 @@ public interface GraphTraversal<S, E> extends Traversal<S, E> {
public default <V> GraphTraversal<S, E> by(final Function<Element, V> elementFunctionProjection,
final Comparator<V> elementFunctionValueComparator) {
- return this.by((Comparator) new ElementFunctionComparator<>(elementFunctionProjection, elementFunctionValueComparator));
+ return ((Function) elementFunctionProjection) instanceof Column ?
+ this.by((Column) ((Function) elementFunctionProjection), elementFunctionValueComparator) :
+ this.by((Comparator) new ElementFunctionComparator<>(elementFunctionProjection, elementFunctionValueComparator));
}
public default <V> GraphTraversal<S, E> by(final String elementPropertyProjection,
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4795042e/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderTest.java
index 4475b7c..b69c625 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderTest.java
@@ -364,7 +364,7 @@ public abstract class OrderTest extends AbstractGremlinProcessTest {
map.put(3, (int) v.get().value("age") * 3);
map.put(4, (int) v.get().value("age"));
return map;
- }).order(Scope.local).by(Column.values, Order.decr).by(Column.keys,Order.incr);
+ }).order(Scope.local).by(Column.values, Order.decr).by(Column.keys, Order.incr);
}
@Override
[14/17] incubator-tinkerpop git commit: GroovyProcessCompiuterSuite
was missing GroovyFlatMapTest. Added it. Added HadoopPool registration to
ToyGraphInputRDD so it doesn't give a WARN message. Also I tweaked
BulkLoaderVertexProgramTest to use target/tes
Posted by ok...@apache.org.
GroovyProcessCompiuterSuite was missing GroovyFlatMapTest. Added it. Added HadoopPool registration to ToyGraphInputRDD so it doesn't give a WARN message. Also I tweaked BulkLoaderVertexProgramTest to use target/test-output/ for its intermediary data.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/f8209353
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/f8209353
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/f8209353
Branch: refs/heads/TINKERPOP3-1014
Commit: f820935303799b8dd661e4fa3686a24a36baad67
Parents: 26dcf3c
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed Dec 2 15:44:50 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Dec 3 12:47:53 2015 -0700
----------------------------------------------------------------------
.../process/GroovyProcessComputerSuite.java | 2 ++
.../BulkLoaderVertexProgramTest.java | 2 +-
.../computer/SparkHadoopGraphProvider.java | 16 +++++----
.../spark/structure/io/ToyGraphInputRDD.java | 38 +++++++++++---------
4 files changed, 33 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f8209353/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java
----------------------------------------------------------------------
diff --git a/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java b/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java
index 9df7902..d07d52f 100644
--- a/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java
+++ b/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java
@@ -47,6 +47,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyAddEdgeTest
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyCoalesceTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyConstantTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyCountTest;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyFlatMapTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyFoldTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyGraphTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyLoopsTest;
@@ -128,6 +129,7 @@ public class GroovyProcessComputerSuite extends ProcessComputerSuite {
GroovyCoalesceTest.Traversals.class,
GroovyConstantTest.Traversals.class,
GroovyCountTest.Traversals.class,
+ GroovyFlatMapTest.Traversals.class,
GroovyFoldTest.Traversals.class,
GroovyGraphTest.Traversals.class,
GroovyLoopsTest.Traversals.class,
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f8209353/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java
index 61a267f..992bfa4 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java
@@ -52,7 +52,7 @@ import static org.junit.Assert.assertTrue;
*/
public class BulkLoaderVertexProgramTest extends AbstractGremlinProcessTest {
- final static String TINKERGRAPH_LOCATION = "/tmp/tinkertest.kryo";
+ final static String TINKERGRAPH_LOCATION = "target/test-output/tinkertest.kryo";
private BulkLoader getBulkLoader(final BulkLoaderVertexProgram blvp) throws Exception {
final Field field = BulkLoaderVertexProgram.class.getDeclaredField("bulkLoader");
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f8209353/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
index 2cfeea3..de5a60d 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
@@ -22,6 +22,8 @@ import org.apache.tinkerpop.gremlin.GraphProvider;
import org.apache.tinkerpop.gremlin.LoadGraphWith;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.HadoopGraphProvider;
+import org.apache.tinkerpop.gremlin.process.computer.bulkloading.BulkLoaderVertexProgramTest;
+import org.apache.tinkerpop.gremlin.process.computer.ranking.PageRankVertexProgramTest;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
import org.apache.tinkerpop.gremlin.spark.structure.io.InputRDDFormat;
@@ -43,13 +45,13 @@ public final class SparkHadoopGraphProvider extends HadoopGraphProvider {
@Override
public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
final Map<String, Object> config = super.getBaseConfiguration(graphName, test, testMethodName, loadGraphWith);
- if (null != loadGraphWith) {
- if (RANDOM.nextBoolean()) {
- config.remove(Constants.GREMLIN_HADOOP_INPUT_LOCATION);
- config.put(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, ToyGraphInputRDD.class.getCanonicalName());
- config.put(ToyGraphInputRDD.GREMLIN_SPARK_TOY_GRAPH, loadGraphWith.toString());
- config.put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName());
- }
+ config.put(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true); // this makes the test suite go really fast
+ if (null != loadGraphWith &&
+ !test.equals(BulkLoaderVertexProgramTest.class) &&
+ !test.equals(PageRankVertexProgramTest.class) &&
+ RANDOM.nextBoolean()) {
+ config.put(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, ToyGraphInputRDD.class.getCanonicalName());
+ config.put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName());
}
/// spark configuration
config.put("spark.master", "local[4]");
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f8209353/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
index ed97c04..ea3636f 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
@@ -22,11 +22,13 @@ package org.apache.tinkerpop.gremlin.spark.structure.io;
import org.apache.commons.configuration.Configuration;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.tinkerpop.gremlin.LoadGraphWith;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPools;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.io.GraphReader;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
@@ -34,36 +36,38 @@ import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
import scala.Tuple2;
import java.io.IOException;
+import java.io.InputStream;
import java.util.List;
-import java.util.stream.Collectors;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
public final class ToyGraphInputRDD implements InputRDD {
- public static final String GREMLIN_SPARK_TOY_GRAPH = "gremlin.spark.toyGraph";
-
@Override
public JavaPairRDD<Object, VertexWritable> readGraphRDD(final Configuration configuration, final JavaSparkContext sparkContext) {
- final List<Vertex> vertices;
- if (configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH).equals(LoadGraphWith.GraphData.MODERN.toString()))
- vertices = IteratorUtils.list(TinkerFactory.createModern().vertices());
- else if (configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH).equals(LoadGraphWith.GraphData.CLASSIC.toString()))
- vertices = IteratorUtils.list(TinkerFactory.createClassic().vertices());
- else if (configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH).equals(LoadGraphWith.GraphData.CREW.toString()))
- vertices = IteratorUtils.list(TinkerFactory.createTheCrew().vertices());
- else if (configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH).equals(LoadGraphWith.GraphData.GRATEFUL.toString())) {
+ HadoopPools.initialize(TinkerGraph.open().configuration());
+ final List<VertexWritable> vertices;
+ if (configuration.getString(Constants.GREMLIN_HADOOP_INPUT_LOCATION).contains("modern"))
+ vertices = IteratorUtils.list(IteratorUtils.map(TinkerFactory.createModern().vertices(), VertexWritable::new));
+ else if (configuration.getString(Constants.GREMLIN_HADOOP_INPUT_LOCATION).contains("classic"))
+ vertices = IteratorUtils.list(IteratorUtils.map(TinkerFactory.createClassic().vertices(), VertexWritable::new));
+ else if (configuration.getString(Constants.GREMLIN_HADOOP_INPUT_LOCATION).contains("crew"))
+ vertices = IteratorUtils.list(IteratorUtils.map(TinkerFactory.createTheCrew().vertices(), VertexWritable::new));
+ else if (configuration.getString(Constants.GREMLIN_HADOOP_INPUT_LOCATION).contains("grateful")) {
try {
final Graph graph = TinkerGraph.open();
- graph.io(GryoIo.build()).readGraph(GryoResourceAccess.class.getResource("grateful-dead.kryo").getFile());
- vertices = IteratorUtils.list(graph.vertices());
+ final GraphReader reader = GryoReader.build().mapper(graph.io(GryoIo.build()).mapper().create()).create();
+ try (final InputStream stream = GryoResourceAccess.class.getResourceAsStream("grateful-dead.kryo")) {
+ reader.readGraph(stream, graph);
+ }
+ vertices = IteratorUtils.list(IteratorUtils.map(graph.vertices(), VertexWritable::new));
} catch (final IOException e) {
throw new IllegalStateException(e.getMessage(), e);
}
} else
- throw new IllegalArgumentException("No legal toy graph was provided to load: " + configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH));
+ throw new IllegalArgumentException("No legal toy graph was provided to load: " + configuration.getProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION));
- return sparkContext.parallelize(vertices.stream().map(VertexWritable::new).collect(Collectors.toList())).mapToPair(vertex -> new Tuple2<>(vertex.get().id(), vertex));
+ return sparkContext.parallelize(vertices).mapToPair(vertex -> new Tuple2<>(vertex.get().id(), vertex));
}
}
[13/17] incubator-tinkerpop git commit: GroovyProcessStartard was
missing GroovyFlatMapTest. Added.
Posted by ok...@apache.org.
GroovyProcessStartard was missing GroovyFlatMapTest. Added.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/5fda89ab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/5fda89ab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/5fda89ab
Branch: refs/heads/TINKERPOP3-1014
Commit: 5fda89ab9cb58a54be5c2622a10d6fad9b942271
Parents: f820935
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed Dec 2 15:49:42 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Dec 3 12:47:53 2015 -0700
----------------------------------------------------------------------
.../tinkerpop/gremlin/process/GroovyProcessStandardSuite.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/5fda89ab/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessStandardSuite.java
----------------------------------------------------------------------
diff --git a/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessStandardSuite.java b/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessStandardSuite.java
index ae91b5f..7024869 100644
--- a/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessStandardSuite.java
+++ b/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessStandardSuite.java
@@ -48,6 +48,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyAddVertexTe
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyCoalesceTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyConstantTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyCountTest;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyFlatMapTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyFoldTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyGraphTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyLoopsTest;
@@ -128,6 +129,7 @@ public class GroovyProcessStandardSuite extends ProcessStandardSuite {
GroovyConstantTest.Traversals.class,
GroovyCountTest.Traversals.class,
GroovyFoldTest.Traversals.class,
+ GroovyFlatMapTest.Traversals.class,
GroovyGraphTest.Traversals.class,
GroovyLoopsTest.Traversals.class,
GroovyMapTest.Traversals.class,
[15/17] incubator-tinkerpop git commit: Introduce the "ci" profile to
Travis
Posted by ok...@apache.org.
Introduce the "ci" profile to Travis
It builds locally with a sizeable reduction in logging.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/b2fee755
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/b2fee755
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/b2fee755
Branch: refs/heads/TINKERPOP3-1014
Commit: b2fee7559d5288779ec622f48b54724b52503eb2
Parents: 4795042
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Thu Dec 3 13:26:07 2015 -0500
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Dec 3 12:48:58 2015 -0700
----------------------------------------------------------------------
.travis.yml | 3 ++-
pom.xml | 10 ++--------
2 files changed, 4 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b2fee755/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index a8f56e5..659b57c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,4 +9,5 @@ addons:
packages:
- oracle-java8-installer
script:
- - "mvn clean install"
\ No newline at end of file
+ - "mvn clean install -Dci"
+
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b2fee755/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index c184a75..0332cdc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -133,7 +133,7 @@ limitations under the License.
the modules under the root directory, not the root
directory itself. -->
<log4j.properties.dir>${project.basedir}/../gremlin-server/conf/</log4j.properties.dir>
- <log4j.properties>log4j-server.properties</log4j.properties>
+ <log4j.properties>log4j.properties</log4j.properties>
<metrics.version>3.0.2</metrics.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
@@ -930,15 +930,9 @@ limitations under the License.
<artifactId>maven-surefire-plugin</artifactId>
<version>2.17</version>
<configuration>
- <argLine>-Dlog4j.configuration=${log4j.properties}
- </argLine>
<systemPropertyVariables>
- <muteTestLogs>false</muteTestLogs>
+ <muteTestLogs>true</muteTestLogs>
</systemPropertyVariables>
- <additionalClasspathElements>
- <!-- So that Log4j can find the Log4j properties file -->
- <additionalClasspathElement>${log4j.properties.dir}</additionalClasspathElement>
- </additionalClasspathElements>
<excludes>
<exclude>**/*IntegrateTest.java</exclude>
<exclude>**/*PerformanceTest.java</exclude>
[06/17] incubator-tinkerpop git commit: Column enum now implements
Function. SelectColumnStep goes away as now select(column) is simply
(un(ddadhe covers) map(column) -- bam. Order.valueDecr/etc. goes away in
favor of order().by(values, decr). This means
Posted by ok...@apache.org.
Column enum now implements Function. SelectColumnStep goes away as now select(column) is simply (un(ddadhe covers) map(column) -- bam. Order.valueDecr/etc. goes away in favor of order().by(values,decr). This means we have decoupled the selector from the comparator so people can do other things besides decr and incr on values.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/b7c6041c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/b7c6041c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/b7c6041c
Branch: refs/heads/TINKERPOP3-1014
Commit: b7c6041cf01fce0258f11faa1e405364e606fbf2
Parents: 0e43bdd
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Nov 24 15:04:02 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Dec 3 12:47:52 2015 -0700
----------------------------------------------------------------------
docs/src/reference/the-traversal.asciidoc | 16 ++---
.../gremlin/process/traversal/Order.java | 28 ++++++--
.../traversal/dsl/graph/GraphTraversal.java | 12 ++--
.../traversal/step/map/SelectColumnStep.java | 68 --------------------
.../traversal/step/util/FunctionComparator.java | 56 ++++++++++++++++
.../tinkerpop/gremlin/structure/Column.java | 37 ++++++++++-
.../process/traversal/step/map/OrderTest.java | 3 +-
.../structure/TinkerGraphPlayTest.java | 7 +-
8 files changed, 137 insertions(+), 90 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b7c6041c/docs/src/reference/the-traversal.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/the-traversal.asciidoc b/docs/src/reference/the-traversal.asciidoc
index 8d0b2fb..6ce4541 100644
--- a/docs/src/reference/the-traversal.asciidoc
+++ b/docs/src/reference/the-traversal.asciidoc
@@ -1573,11 +1573,11 @@ ranking.
graph.io(graphml()).readGraph('data/grateful-dead.xml')
g = graph.traversal()
g.V().hasLabel('song').out('followedBy').groupCount().by('name').
- order(local).by(valueDecr).limit(local, 5)
+ order(local).by(values,decr).limit(local, 5)
g.V().hasLabel('song').out('followedBy').groupCount().by('name').
- order(local).by(valueDecr).limit(local, 5).select(keys)
+ order(local).by(values,decr).limit(local, 5).select(keys)
g.V().hasLabel('song').out('followedBy').groupCount().by('name').
- order(local).by(valueDecr).limit(local, 5).select(keys).unfold()
+ order(local).by(values,decr).limit(local, 5).select(keys).unfold()
----
Similarly, for extracting the values from a path or map.
@@ -1589,7 +1589,7 @@ g = graph.traversal()
g.V().hasLabel('song').out('sungBy').groupCount().by('name') <1>
g.V().hasLabel('song').out('sungBy').groupCount().by('name').select(values) <2>
g.V().hasLabel('song').out('sungBy').groupCount().by('name').select(values).unfold().
- groupCount().order(local).by(valueDecr).limit(local, 5) <3>
+ groupCount().order(local).by(values,decr).limit(local, 5) <3>
----
<1> Which artist sung how many songs?
@@ -1779,10 +1779,10 @@ that can be used to time execution of a body of code.
[gremlin-groovy,modern]
----
-g.V().repeat(both().groupCount('m')).times(16).cap('m').order(local).by(valueDecr).next()
-clock(1) {g.V().repeat(both().groupCount('m')).times(16).cap('m').order(local).by(valueDecr).next()}
-g.V().repeat(timeLimit(2).both().groupCount('m')).times(16).cap('m').order(local).by(valueDecr).next()
-clock(1) {g.V().repeat(timeLimit(2).both().groupCount('m')).times(16).cap('m').order(local).by(valueDecr).next()}
+g.V().repeat(both().groupCount('m')).times(16).cap('m').order(local).by(values,decr).next()
+clock(1) {g.V().repeat(both().groupCount('m')).times(16).cap('m').order(local).by(values,decr).next()}
+g.V().repeat(timeLimit(2).both().groupCount('m')).times(16).cap('m').order(local).by(values,decr).next()
+clock(1) {g.V().repeat(timeLimit(2).both().groupCount('m')).times(16).cap('m').order(local).by(values,decr).next()}
----
In essence, the relative order is respected, even through the number of traversers at each vertex is not. The primary
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b7c6041c/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/Order.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/Order.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/Order.java
index e04863b..4e3801a 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/Order.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/Order.java
@@ -47,7 +47,12 @@ public enum Order implements Comparator<Object> {
public Order reversed() {
return incr;
}
- }, keyIncr {
+ },
+ /**
+ * @deprecated Use {@link org.apache.tinkerpop.gremlin.structure.Column#keys};
+ */
+ @Deprecated
+ keyIncr {
@Override
public int compare(final Object first, final Object second) {
return Comparator.<Comparable>naturalOrder().compare(((Map.Entry<Comparable, ?>) first).getKey(), ((Map.Entry<Comparable, ?>) second).getKey());
@@ -57,7 +62,12 @@ public enum Order implements Comparator<Object> {
public Order reversed() {
return keyDecr;
}
- }, valueIncr {
+ },
+ /**
+ * @deprecated Use {@link org.apache.tinkerpop.gremlin.structure.Column#values};
+ */
+ @Deprecated
+ valueIncr {
@Override
public int compare(final Object first, final Object second) {
return Comparator.<Comparable>naturalOrder().compare(((Map.Entry<?, Comparable>) first).getValue(), ((Map.Entry<?, Comparable>) second).getValue());
@@ -67,7 +77,12 @@ public enum Order implements Comparator<Object> {
public Order reversed() {
return valueDecr;
}
- }, keyDecr {
+ },
+ /**
+ * @deprecated Use {@link org.apache.tinkerpop.gremlin.structure.Column#keys};
+ */
+ @Deprecated
+ keyDecr {
@Override
public int compare(final Object first, final Object second) {
return Comparator.<Comparable>reverseOrder().compare(((Map.Entry<Comparable, ?>) first).getKey(), ((Map.Entry<Comparable, ?>) second).getKey());
@@ -77,7 +92,12 @@ public enum Order implements Comparator<Object> {
public Order reversed() {
return keyIncr;
}
- }, valueDecr {
+ },
+ /**
+ * @deprecated Use {@link org.apache.tinkerpop.gremlin.structure.Column#values};
+ */
+ @Deprecated
+ valueDecr {
@Override
public int compare(final Object first, final Object second) {
return Comparator.<Comparable>reverseOrder().compare(((Map.Entry<?, Comparable>) first).getValue(), ((Map.Entry<?, Comparable>) second).getValue());
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b7c6041c/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java
index 97d0c53..3673bce 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java
@@ -99,7 +99,6 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertyValueStep
import org.apache.tinkerpop.gremlin.process.traversal.step.map.RangeLocalStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.SackStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.SampleLocalStep;
-import org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectColumnStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectOneStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.SumGlobalStep;
@@ -128,6 +127,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.TraversalS
import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.TreeSideEffectStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.ElementFunctionComparator;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.ElementValueComparator;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.FunctionComparator;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.HasContainer;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.TraversalComparator;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.Tree;
@@ -472,11 +472,11 @@ public interface GraphTraversal<S, E> extends Traversal<S, E> {
}
public default <E2> GraphTraversal<S, Collection<E2>> select(final Column column) {
- return this.asAdmin().addStep(new SelectColumnStep<>(this.asAdmin(), column));
+ return this.map(new FunctionTraverser<>((Function) column));
}
/**
- * @deprecated As of release 3.1.0, replaced by {@link #select(SelectColumnStep.Column)}
+ * @deprecated As of release 3.1.0, replaced by {@link GraphTraversal#select(Column)}
*/
@Deprecated
public default <E2> GraphTraversal<S, E2> mapValues() {
@@ -484,7 +484,7 @@ public interface GraphTraversal<S, E> extends Traversal<S, E> {
}
/**
- * @deprecated As of release 3.1.0, replaced by {@link #select(SelectColumnStep.Column)}
+ * @deprecated As of release 3.1.0, replaced by {@link GraphTraversal#select(Column)}
*/
@Deprecated
public default <E2> GraphTraversal<S, E2> mapKeys() {
@@ -1198,6 +1198,10 @@ public interface GraphTraversal<S, E> extends Traversal<S, E> {
return this.by((Comparator) order);
}
+ public default <V> GraphTraversal<S, E> by(final Column column, final Comparator<V> objectComparator) {
+ return this.by(new FunctionComparator(column, objectComparator));
+ }
+
public default <V> GraphTraversal<S, E> by(final Function<Element, V> elementFunctionProjection,
final Comparator<V> elementFunctionValueComparator) {
return this.by((Comparator) new ElementFunctionComparator<>(elementFunctionProjection, elementFunctionValueComparator));
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b7c6041c/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/SelectColumnStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/SelectColumnStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/SelectColumnStep.java
deleted file mode 100644
index 1c28003..0000000
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/SelectColumnStep.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.process.traversal.step.map;
-
-import org.apache.tinkerpop.gremlin.structure.Column;
-import org.apache.tinkerpop.gremlin.process.traversal.Path;
-import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
-import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
-import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
-import org.apache.tinkerpop.gremlin.util.function.ArrayListSupplier;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class SelectColumnStep<S, E> extends MapStep<S, Collection<E>> {
-
- private final Column column;
-
- public SelectColumnStep(final Traversal.Admin traversal, final Column column) {
- super(traversal);
- this.column = column;
- }
-
- @Override
- protected Collection<E> map(Traverser.Admin<S> traverser) {
- final S start = traverser.get();
- if (start instanceof Map)
- return this.column.equals(Column.keys) ? ((Map<E, ?>) start).keySet() : ((Map<?, E>) start).values();
- else if (start instanceof Path)
- return (Collection<E>) (this.column.equals(Column.keys) ? new ArrayList<>(((Path) start).labels()) : new ArrayList(((Path) start).objects())); // necessary for serialization in complex GraphComputers (find fix)
- else if (start instanceof Map.Entry) // TODO: remove support for this?
- return Collections.singleton(this.column.equals(Column.keys) ? ((Map.Entry<E, ?>) start).getKey() : ((Map.Entry<?, E>) start).getValue());
- else
- throw new IllegalStateException("The traverser does not reference a map or path: " + traverser);
- }
-
- @Override
- public String toString() {
- return StringFactory.stepString(this, this.column);
- }
-
- @Override
- public int hashCode() {
- return super.hashCode() ^ this.column.hashCode();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b7c6041c/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/FunctionComparator.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/FunctionComparator.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/FunctionComparator.java
new file mode 100644
index 0000000..a133e38
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/FunctionComparator.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.process.traversal.step.util;
+
+import java.io.Serializable;
+import java.util.Comparator;
+import java.util.function.Function;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class FunctionComparator<A, B> implements Comparator<A>, Serializable {
+
+ private final Function<A, B> function;
+ private final Comparator<B> comparator;
+
+ public FunctionComparator(final Function<A, B> function, final Comparator<B> comparator) {
+ this.function = function;
+ this.comparator = comparator;
+ }
+
+ public Function<A, B> getFunction() {
+ return this.function;
+ }
+
+ public Comparator<B> getComparator() {
+ return this.comparator;
+ }
+
+ @Override
+ public int compare(final A first, final A second) {
+ return this.comparator.compare(this.function.apply(first), this.function.apply(second));
+ }
+
+ @Override
+ public String toString() {
+ return this.comparator.toString() + "(" + this.function + ')';
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b7c6041c/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Column.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Column.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Column.java
index 1ab49fa..fca90d7 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Column.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Column.java
@@ -19,19 +19,50 @@
package org.apache.tinkerpop.gremlin.structure;
+import org.apache.tinkerpop.gremlin.process.traversal.Path;
+
+import java.util.Map;
+import java.util.function.Function;
+
/**
* Column references a particular type of column in a complex data structure such as a {@link java.util.Map} or a {@link org.apache.tinkerpop.gremlin.process.traversal.Path}.
*
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-public enum Column {
+public enum Column implements Function<Object, Object> {
/**
* The keys associated with the data structure.
*/
- keys,
+ keys {
+ @Override
+ public Object apply(final Object object) {
+ if (object instanceof Map)
+ return ((Map) object).keySet();
+ else if (object instanceof Map.Entry)
+ return ((Map.Entry) object).getKey();
+ else if (object instanceof Path)
+ return ((Path) object).labels();
+ else
+ throw new IllegalArgumentException("The provided object does not have accessible keys: " + object.getClass());
+ }
+ },
/**
* The values associated with the data structure.
*/
- values
+ values {
+ @Override
+ public Object apply(final Object object) {
+ if (object instanceof Map)
+ return ((Map) object).values();
+ else if (object instanceof Map.Entry)
+ return ((Map.Entry) object).getValue();
+ else if (object instanceof Path)
+ return ((Path) object).objects();
+ else
+ throw new IllegalArgumentException("The provided object does not have accessible keys: " + object.getClass());
+ }
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b7c6041c/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderTest.java
index 2eec7ac..3f08d60 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderTest.java
@@ -27,6 +27,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.Scope;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalEngine;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.apache.tinkerpop.gremlin.structure.Column;
import org.apache.tinkerpop.gremlin.structure.T;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.junit.Test;
@@ -363,7 +364,7 @@ public abstract class OrderTest extends AbstractGremlinProcessTest {
map.put(3, (int) v.get().value("age") * 3);
map.put(4, (int) v.get().value("age"));
return map;
- }).order(Scope.local).by(Order.valueDecr).by(Order.keyIncr);
+ }).order(Scope.local).by(Column.values, Order.decr).by(Column.keys,Order.incr);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b7c6041c/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphPlayTest.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphPlayTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphPlayTest.java
index fdb44ad..64c2e84 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphPlayTest.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphPlayTest.java
@@ -20,13 +20,14 @@ package org.apache.tinkerpop.gremlin.tinkergraph.structure;
import org.apache.tinkerpop.gremlin.process.computer.bulkloading.BulkLoaderVertexProgram;
import org.apache.tinkerpop.gremlin.process.traversal.Operator;
+import org.apache.tinkerpop.gremlin.process.traversal.Order;
import org.apache.tinkerpop.gremlin.process.traversal.P;
import org.apache.tinkerpop.gremlin.process.traversal.Scope;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
-import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalMetrics;
+import org.apache.tinkerpop.gremlin.structure.Column;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.T;
import org.apache.tinkerpop.gremlin.structure.Vertex;
@@ -39,6 +40,7 @@ import org.junit.Test;
import java.io.File;
import java.util.Arrays;
import java.util.List;
+import java.util.function.Function;
import java.util.function.Supplier;
import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.*;
@@ -53,7 +55,8 @@ public class TinkerGraphPlayTest {
public void testPlay8() throws Exception {
Graph graph = TinkerFactory.createModern();
GraphTraversalSource g = graph.traversal(); //GraphTraversalSource.computer());
- System.out.println(g.V().outE("knows").identity().inV().count().is(P.eq(5)).explain());
+ //System.out.println(g.V().outE("knows").identity().inV().count().is(P.eq(5)).explain());
+ System.out.println(g.V().both().both().groupCount().order(Scope.local).by(Column.values, Order.incr).toList());
}
[16/17] incubator-tinkerpop git commit: Updated CHANGELOG.
Posted by ok...@apache.org.
Updated CHANGELOG.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/1134ad2a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/1134ad2a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/1134ad2a
Branch: refs/heads/TINKERPOP3-1014
Commit: 1134ad2a50ae4cce7fbcd2b8f362ff762139c7e4
Parents: b2fee75
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Thu Dec 3 12:01:03 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Dec 3 12:48:58 2015 -0700
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1134ad2a/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 869b494..717aa71 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,6 +26,7 @@ image::https://raw.githubusercontent.com/apache/incubator-tinkerpop/master/docs/
TinkerPop 3.1.1 (NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* `Column` now implements `Function` and now `by(valueDecr)` can be replaced with `by(values,decr)` and thus, projection and order are now separated.
* Added `InputRDDFormat` which wraps an `InputRDD` to make it accessible to Hadoop and not just Spark.
* Added `AbstractSparkTest` which handles closing `SparkContext` instances between tests now that we support persisted contexts.
* Fixed a serialization bug in `GryoSerializer` that made it difficult for graph providers to yield `InputRDDs` for `SparkGraphComputer`.
[04/17] incubator-tinkerpop git commit: some organization and clean
up. Stuff is lookin SOLID. Time to run full integration tests.
Posted by ok...@apache.org.
some organization and clean up. Stuff is lookin SOLID. Time to run full integration tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/26dcf3c9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/26dcf3c9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/26dcf3c9
Branch: refs/heads/TINKERPOP3-1014
Commit: 26dcf3c943bf1328747cd78d6fd5a814dfe6a5da
Parents: ddbbec0
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed Dec 2 13:11:29 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Dec 3 12:47:52 2015 -0700
----------------------------------------------------------------------
.../process/computer/SparkContextHelper.java | 39 +++++++++++
.../process/computer/SparkGraphComputer.java | 15 ++---
.../spark/structure/io/InputRDDFormat.java | 8 +--
.../process/computer/LocalPropertyTest.java | 4 +-
.../computer/SparkHadoopGraphProvider.java | 38 +++--------
.../spark/structure/io/ClassicInputRDD.java | 39 -----------
.../spark/structure/io/GratefulInputRDD.java | 50 --------------
.../spark/structure/io/InputOutputRDDTest.java | 1 -
.../spark/structure/io/InputRDDTest.java | 1 -
.../spark/structure/io/ModernInputRDD.java | 41 ------------
.../spark/structure/io/OutputRDDTest.java | 1 -
.../io/PersistedInputOutputRDDTest.java | 10 +--
.../spark/structure/io/TheCrewInputRDD.java | 39 -----------
.../spark/structure/io/ToyGraphInputRDD.java | 69 ++++++++++++++++++++
14 files changed, 134 insertions(+), 221 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26dcf3c9/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkContextHelper.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkContextHelper.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkContextHelper.java
new file mode 100644
index 0000000..ed553c5
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkContextHelper.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.process.computer;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class SparkContextHelper {
+
+ private SparkContextHelper() {
+
+ }
+
+ public static void tryToCloseContext(final JavaSparkContext context, final Configuration configuration) {
+ if (context != null && !configuration.getBoolean(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, false))
+ context.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26dcf3c9/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
index e7566d5..c20f1b0 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
@@ -228,8 +228,7 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
finalMemory.setRuntime(System.currentTimeMillis() - startTime);
return new DefaultComputerResult(InputOutputHelper.getOutputGraph(apacheConfiguration, this.resultGraph, this.persist), finalMemory.asImmutable());
} finally {
- if (sparkContext != null && !apacheConfiguration.getBoolean(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, false))
- sparkContext.stop();
+ SparkContextHelper.tryToCloseContext(sparkContext, apacheConfiguration);
}
}, exec);
}
@@ -268,14 +267,14 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
* Execution rather than applying the entire configuration.
*/
final String[] validPropertyNames = {
- "spark.job.description",
- "spark.jobGroup.id",
- "spark.job.interruptOnCancel",
- "spark.scheduler.pool"
+ "spark.job.description",
+ "spark.jobGroup.id",
+ "spark.job.interruptOnCancel",
+ "spark.scheduler.pool"
};
- for (String propertyName: validPropertyNames){
- if (sparkConfiguration.contains(propertyName)){
+ for (String propertyName : validPropertyNames) {
+ if (sparkConfiguration.contains(propertyName)) {
String propertyValue = sparkConfiguration.get(propertyName);
this.logger.info("Setting Thread Local SparkContext Property - "
+ propertyName + " : " + propertyValue);
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26dcf3c9/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDFormat.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDFormat.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDFormat.java
index 3952c66..d1a198e 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDFormat.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDFormat.java
@@ -31,6 +31,7 @@ import org.apache.spark.api.java.JavaSparkContext;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
+import org.apache.tinkerpop.gremlin.spark.process.computer.SparkContextHelper;
import scala.Tuple2;
import java.io.IOException;
@@ -70,8 +71,8 @@ public final class InputRDDFormat extends InputFormat<NullWritable, VertexWritab
final SparkConf sparkConfiguration = new SparkConf();
sparkConfiguration.setAppName(UUID.randomUUID().toString());
hadoopConfiguration.forEach(entry -> sparkConfiguration.set(entry.getKey(), entry.getValue()));
- InputRDD inputRDD = (InputRDD) Class.forName(sparkConfiguration.get(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD)).newInstance();
- JavaSparkContext javaSparkContext = new JavaSparkContext(SparkContext.getOrCreate(sparkConfiguration));
+ final InputRDD inputRDD = (InputRDD) Class.forName(sparkConfiguration.get(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD)).newInstance();
+ final JavaSparkContext javaSparkContext = new JavaSparkContext(SparkContext.getOrCreate(sparkConfiguration));
final Iterator<Tuple2<Object, VertexWritable>> iterator = inputRDD.readGraphRDD(ConfUtil.makeApacheConfiguration(taskAttemptContext.getConfiguration()), javaSparkContext).toLocalIterator();
return new RecordReader<NullWritable, VertexWritable>() {
@Override
@@ -101,8 +102,7 @@ public final class InputRDDFormat extends InputFormat<NullWritable, VertexWritab
@Override
public void close() throws IOException {
- if (!hadoopConfiguration.getBoolean(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, false))
- javaSparkContext.close();
+ SparkContextHelper.tryToCloseContext(javaSparkContext, ConfUtil.makeApacheConfiguration(hadoopConfiguration));
}
};
} catch (final ClassNotFoundException | InstantiationException | IllegalAccessException e) {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26dcf3c9/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/LocalPropertyTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/LocalPropertyTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/LocalPropertyTest.java
index e0fe796..671bee8 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/LocalPropertyTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/LocalPropertyTest.java
@@ -25,7 +25,6 @@ import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.JavaSparkStatusTracker;
-import org.apache.spark.serializer.KryoSerializer;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
@@ -36,6 +35,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSo
import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
import org.apache.tinkerpop.gremlin.spark.structure.io.PersistedInputRDD;
import org.apache.tinkerpop.gremlin.spark.structure.io.PersistedOutputRDD;
+import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
import org.junit.Test;
@@ -53,7 +53,7 @@ public class LocalPropertyTest {
final String rddName = "target/test-output/" + UUID.randomUUID();
final Configuration configuration = new BaseConfiguration();
configuration.setProperty("spark.master", "local[4]");
- configuration.setProperty("spark.serializer", KryoSerializer.class.getCanonicalName());
+ configuration.setProperty("spark.serializer", GryoSerializer.class.getCanonicalName());
configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, GryoInputFormat.class.getCanonicalName());
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26dcf3c9/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
index 2328176..2cfeea3 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
@@ -24,11 +24,9 @@ import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.HadoopGraphProvider;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
-import org.apache.tinkerpop.gremlin.spark.structure.io.ClassicInputRDD;
-import org.apache.tinkerpop.gremlin.spark.structure.io.GratefulInputRDD;
import org.apache.tinkerpop.gremlin.spark.structure.io.InputRDDFormat;
-import org.apache.tinkerpop.gremlin.spark.structure.io.ModernInputRDD;
-import org.apache.tinkerpop.gremlin.spark.structure.io.TheCrewInputRDD;
+import org.apache.tinkerpop.gremlin.spark.structure.io.ToyGraphInputRDD;
+import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
import org.apache.tinkerpop.gremlin.structure.Graph;
import java.util.Map;
@@ -46,36 +44,16 @@ public final class SparkHadoopGraphProvider extends HadoopGraphProvider {
public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
final Map<String, Object> config = super.getBaseConfiguration(graphName, test, testMethodName, loadGraphWith);
if (null != loadGraphWith) {
- if (loadGraphWith.equals(LoadGraphWith.GraphData.MODERN)) {
- if (RANDOM.nextBoolean()) {
- config.remove(Constants.GREMLIN_HADOOP_INPUT_LOCATION);
- config.put(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, ModernInputRDD.class.getCanonicalName());
- config.put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName());
- }
- } else if (loadGraphWith.equals(LoadGraphWith.GraphData.CREW)) {
- if (RANDOM.nextBoolean()) {
- config.remove(Constants.GREMLIN_HADOOP_INPUT_LOCATION);
- config.put(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, TheCrewInputRDD.class.getCanonicalName());
- config.put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName());
- }
- } else if (loadGraphWith.equals(LoadGraphWith.GraphData.CLASSIC)) {
- if (RANDOM.nextBoolean()) {
- config.remove(Constants.GREMLIN_HADOOP_INPUT_LOCATION);
- config.put(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, ClassicInputRDD.class.getCanonicalName());
- config.put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName());
- }
- } else if (loadGraphWith.equals(LoadGraphWith.GraphData.GRATEFUL)) {
- if (RANDOM.nextBoolean()) {
- config.remove(Constants.GREMLIN_HADOOP_INPUT_LOCATION);
- config.put(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, GratefulInputRDD.class.getCanonicalName());
- config.put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName());
- }
+ if (RANDOM.nextBoolean()) {
+ config.remove(Constants.GREMLIN_HADOOP_INPUT_LOCATION);
+ config.put(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, ToyGraphInputRDD.class.getCanonicalName());
+ config.put(ToyGraphInputRDD.GREMLIN_SPARK_TOY_GRAPH, loadGraphWith.toString());
+ config.put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName());
}
}
/// spark configuration
config.put("spark.master", "local[4]");
- //config.put("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
- config.put("spark.serializer", "org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer");
+ config.put("spark.serializer", GryoSerializer.class.getCanonicalName());
config.put("spark.kryo.registrationRequired", true);
return config;
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26dcf3c9/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ClassicInputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ClassicInputRDD.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ClassicInputRDD.java
deleted file mode 100644
index 4512b61..0000000
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ClassicInputRDD.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.spark.structure.io;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory;
-import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
-import scala.Tuple2;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class ClassicInputRDD implements InputRDD {
-
- @Override
- public JavaPairRDD<Object, VertexWritable> readGraphRDD(final Configuration configuration, final JavaSparkContext sparkContext) {
- return sparkContext.parallelize(IteratorUtils.list(IteratorUtils.map(TinkerFactory.createClassic().vertices(), VertexWritable::new))).mapToPair(vertex -> new Tuple2<>(vertex.get().id(), vertex));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26dcf3c9/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/GratefulInputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/GratefulInputRDD.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/GratefulInputRDD.java
deleted file mode 100644
index 396aa75..0000000
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/GratefulInputRDD.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.spark.structure.io;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess;
-import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
-import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
-import scala.Tuple2;
-
-import java.io.IOException;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GratefulInputRDD implements InputRDD {
-
- @Override
- public JavaPairRDD<Object, VertexWritable> readGraphRDD(final Configuration configuration, final JavaSparkContext sparkContext) {
- try {
- final Graph graph = TinkerGraph.open();
- graph.io(GryoIo.build()).readGraph(GryoResourceAccess.class.getResource("grateful-dead.kryo").getFile());
- return sparkContext.parallelize(IteratorUtils.list(IteratorUtils.map(graph.vertices(), VertexWritable::new))).mapToPair(vertex -> new Tuple2<>(vertex.get().id(), vertex));
- } catch (final IOException e) {
- throw new IllegalStateException(e.getMessage(), e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26dcf3c9/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputRDDTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputRDDTest.java
index ea62114..50a43bc 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputRDDTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputRDDTest.java
@@ -20,7 +20,6 @@ package org.apache.tinkerpop.gremlin.spark.structure.io;
import org.apache.commons.configuration.BaseConfiguration;
import org.apache.commons.configuration.Configuration;
-import org.apache.spark.serializer.KryoSerializer;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26dcf3c9/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
index 2cbfd66..b64139c 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
@@ -20,7 +20,6 @@ package org.apache.tinkerpop.gremlin.spark.structure.io;
import org.apache.commons.configuration.BaseConfiguration;
import org.apache.commons.configuration.Configuration;
-import org.apache.spark.serializer.KryoSerializer;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26dcf3c9/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ModernInputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ModernInputRDD.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ModernInputRDD.java
deleted file mode 100644
index 849e3e6..0000000
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ModernInputRDD.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.spark.structure.io;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory;
-import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
-import scala.Tuple2;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class ModernInputRDD implements InputRDD {
-
- @Override
- public JavaPairRDD<Object, VertexWritable> readGraphRDD(final Configuration configuration, final JavaSparkContext sparkContext) {
- return sparkContext.
- parallelize(IteratorUtils.list(IteratorUtils.map(TinkerFactory.createModern().vertices(), VertexWritable::new))).
- mapToPair(vertex -> new Tuple2<>(vertex.get().id(), vertex));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26dcf3c9/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java
index f9b6f39..60790e7 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java
@@ -20,7 +20,6 @@ package org.apache.tinkerpop.gremlin.spark.structure.io;
import org.apache.commons.configuration.BaseConfiguration;
import org.apache.commons.configuration.Configuration;
-import org.apache.spark.serializer.KryoSerializer;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26dcf3c9/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java
index 6aeb864..1de2b47 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java
@@ -24,7 +24,6 @@ import org.apache.commons.configuration.Configuration;
import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.serializer.KryoSerializer;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
@@ -37,6 +36,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSo
import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkHadoopGraphProvider;
+import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.io.IoCore;
import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
@@ -58,7 +58,7 @@ public class PersistedInputOutputRDDTest {
final String rddName = "target/test-output/" + UUID.randomUUID();
final Configuration configuration = new BaseConfiguration();
configuration.setProperty("spark.master", "local[4]");
- configuration.setProperty("spark.serializer", KryoSerializer.class.getCanonicalName());
+ configuration.setProperty("spark.serializer", GryoSerializer.class.getCanonicalName());
configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, GryoInputFormat.class.getCanonicalName());
@@ -87,7 +87,7 @@ public class PersistedInputOutputRDDTest {
final String rddName = "target/test-output/" + UUID.randomUUID();
final Configuration configuration = new BaseConfiguration();
configuration.setProperty("spark.master", "local[4]");
- configuration.setProperty("spark.serializer", KryoSerializer.class.getCanonicalName());
+ configuration.setProperty("spark.serializer", GryoSerializer.class.getCanonicalName());
configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, GryoInputFormat.class.getCanonicalName());
@@ -129,7 +129,7 @@ public class PersistedInputOutputRDDTest {
final String rddName = "target/test-output/" + UUID.randomUUID().toString();
final Configuration readConfiguration = new BaseConfiguration();
readConfiguration.setProperty("spark.master", "local[4]");
- readConfiguration.setProperty("spark.serializer", KryoSerializer.class.getCanonicalName());
+ readConfiguration.setProperty("spark.serializer", GryoSerializer.class.getCanonicalName());
readConfiguration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
readConfiguration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, GryoInputFormat.class.getCanonicalName());
readConfiguration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
@@ -176,7 +176,7 @@ public class PersistedInputOutputRDDTest {
final String rddName = "target/test-output/" + UUID.randomUUID().toString();
final Configuration readConfiguration = new BaseConfiguration();
readConfiguration.setProperty("spark.master", "local[4]");
- readConfiguration.setProperty("spark.serializer", KryoSerializer.class.getCanonicalName());
+ readConfiguration.setProperty("spark.serializer", GryoSerializer.class.getCanonicalName());
readConfiguration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
readConfiguration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, GryoInputFormat.class.getCanonicalName());
readConfiguration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26dcf3c9/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/TheCrewInputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/TheCrewInputRDD.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/TheCrewInputRDD.java
deleted file mode 100644
index ff5a274..0000000
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/TheCrewInputRDD.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.spark.structure.io;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory;
-import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
-import scala.Tuple2;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class TheCrewInputRDD implements InputRDD {
-
- @Override
- public JavaPairRDD<Object, VertexWritable> readGraphRDD(final Configuration configuration, final JavaSparkContext sparkContext) {
- return sparkContext.parallelize(IteratorUtils.list(IteratorUtils.map(TinkerFactory.createTheCrew().vertices(), VertexWritable::new))).mapToPair(vertex -> new Tuple2<>(vertex.get().id(), vertex));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26dcf3c9/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
new file mode 100644
index 0000000..ed97c04
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.structure.io;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.tinkerpop.gremlin.LoadGraphWith;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+import scala.Tuple2;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class ToyGraphInputRDD implements InputRDD {
+
+ public static final String GREMLIN_SPARK_TOY_GRAPH = "gremlin.spark.toyGraph";
+
+ @Override
+ public JavaPairRDD<Object, VertexWritable> readGraphRDD(final Configuration configuration, final JavaSparkContext sparkContext) {
+ final List<Vertex> vertices;
+ if (configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH).equals(LoadGraphWith.GraphData.MODERN.toString()))
+ vertices = IteratorUtils.list(TinkerFactory.createModern().vertices());
+ else if (configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH).equals(LoadGraphWith.GraphData.CLASSIC.toString()))
+ vertices = IteratorUtils.list(TinkerFactory.createClassic().vertices());
+ else if (configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH).equals(LoadGraphWith.GraphData.CREW.toString()))
+ vertices = IteratorUtils.list(TinkerFactory.createTheCrew().vertices());
+ else if (configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH).equals(LoadGraphWith.GraphData.GRATEFUL.toString())) {
+ try {
+ final Graph graph = TinkerGraph.open();
+ graph.io(GryoIo.build()).readGraph(GryoResourceAccess.class.getResource("grateful-dead.kryo").getFile());
+ vertices = IteratorUtils.list(graph.vertices());
+ } catch (final IOException e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ } else
+ throw new IllegalArgumentException("No legal toy graph was provided to load: " + configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH));
+
+ return sparkContext.parallelize(vertices.stream().map(VertexWritable::new).collect(Collectors.toList())).mapToPair(vertex -> new Tuple2<>(vertex.get().id(), vertex));
+ }
+}
[12/17] incubator-tinkerpop git commit: updated CHANGELOG again.
Posted by ok...@apache.org.
updated CHANGELOG again.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/447d7056
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/447d7056
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/447d7056
Branch: refs/heads/TINKERPOP3-1014
Commit: 447d7056ef84865efe76d09033f4bd13b1ec0884
Parents: 461be03
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Thu Dec 3 07:51:22 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Dec 3 12:47:53 2015 -0700
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/447d7056/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 90c9f27..869b494 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,6 +26,7 @@ image::https://raw.githubusercontent.com/apache/incubator-tinkerpop/master/docs/
TinkerPop 3.1.1 (NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* Added `InputRDDFormat` which wraps an `InputRDD` to make it accessible to Hadoop and not just Spark.
* Added `AbstractSparkTest` which handles closing `SparkContext` instances between tests now that we support persisted contexts.
* Fixed a serialization bug in `GryoSerializer` that made it difficult for graph providers to yield `InputRDDs` for `SparkGraphComputer`.
* `SparkGraphComputer` is now tested against Gryo, GraphSON, and `InputRDD` data sources.
[09/17] incubator-tinkerpop git commit: Random exceptions started
popping up. I realized that I should kill any residual SparkContext before
and after every test. Added AbstractSparkTest which has a Before/After
annotated method which does the job. No ra
Posted by ok...@apache.org.
Random exceptions started popping up. I realized that I should kill any residual SparkContext before and after every test. Added AbstractSparkTest which has a Before/After annotated method which does the job. No random exceptions now. Thank god.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/87382a9e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/87382a9e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/87382a9e
Branch: refs/heads/TINKERPOP3-1014
Commit: 87382a9e2945cac83e8e422a9525e599688e2c34
Parents: 5fda89a
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed Dec 2 18:35:58 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Dec 3 12:47:53 2015 -0700
----------------------------------------------------------------------
.../gremlin/spark/AbstractSparkTest.java | 43 ++++++++++++++++++++
.../process/computer/LocalPropertyTest.java | 4 +-
.../spark/structure/io/InputOutputRDDTest.java | 3 +-
.../spark/structure/io/InputRDDTest.java | 3 +-
.../spark/structure/io/OutputRDDTest.java | 3 +-
.../io/PersistedInputOutputRDDTest.java | 4 +-
6 files changed, 53 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/87382a9e/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/AbstractSparkTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/AbstractSparkTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/AbstractSparkTest.java
new file mode 100644
index 0000000..02b0e18
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/AbstractSparkTest.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark;
+
+import org.apache.spark.SparkConf;
+import org.apache.spark.SparkContext;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.junit.After;
+import org.junit.Before;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public abstract class AbstractSparkTest {
+
+ @After
+ @Before
+ public void setupTest() {
+ SparkConf sparkConfiguration = new SparkConf();
+ sparkConfiguration.setAppName(this.getClass().getCanonicalName() + "-setupTest");
+ sparkConfiguration.set("spark.master", "local[4]");
+ JavaSparkContext sparkContext = new JavaSparkContext(SparkContext.getOrCreate(sparkConfiguration));
+ sparkContext.close();
+ System.out.println("SparkContext has been closed for " + this.getClass().getCanonicalName() + "-setupTest");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/87382a9e/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/LocalPropertyTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/LocalPropertyTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/LocalPropertyTest.java
index 671bee8..b27b7b9 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/LocalPropertyTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/LocalPropertyTest.java
@@ -33,6 +33,7 @@ import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
+import org.apache.tinkerpop.gremlin.spark.AbstractSparkTest;
import org.apache.tinkerpop.gremlin.spark.structure.io.PersistedInputRDD;
import org.apache.tinkerpop.gremlin.spark.structure.io.PersistedOutputRDD;
import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
@@ -44,8 +45,7 @@ import java.util.UUID;
import static org.junit.Assert.assertTrue;
-public class LocalPropertyTest {
-
+public class LocalPropertyTest extends AbstractSparkTest {
@Test
public void shouldSetThreadLocalProperties() throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/87382a9e/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputRDDTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputRDDTest.java
index 50a43bc..cd78d7d 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputRDDTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputRDDTest.java
@@ -26,6 +26,7 @@ import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
+import org.apache.tinkerpop.gremlin.spark.AbstractSparkTest;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
import org.apache.tinkerpop.gremlin.structure.Graph;
@@ -35,7 +36,7 @@ import org.junit.Test;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-public class InputOutputRDDTest {
+public class InputOutputRDDTest extends AbstractSparkTest {
@Test
public void shouldReadFromWriteToArbitraryRDD() throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/87382a9e/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
index b64139c..1a54af0 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
@@ -25,6 +25,7 @@ import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
import org.apache.tinkerpop.gremlin.process.traversal.P;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.spark.AbstractSparkTest;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
import org.apache.tinkerpop.gremlin.structure.Graph;
@@ -36,7 +37,7 @@ import static org.junit.Assert.assertEquals;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-public class InputRDDTest {
+public class InputRDDTest extends AbstractSparkTest {
@Test
public void shouldReadFromArbitraryRDD() {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/87382a9e/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java
index 60790e7..2ccf0ca 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java
@@ -27,6 +27,7 @@ import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
+import org.apache.tinkerpop.gremlin.spark.AbstractSparkTest;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkHadoopGraphProvider;
import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
@@ -37,7 +38,7 @@ import org.junit.Test;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-public class OutputRDDTest {
+public class OutputRDDTest extends AbstractSparkTest {
@Test
public void shouldWriteToArbitraryRDD() throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/87382a9e/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java
index 1de2b47..bc55c04 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java
@@ -34,6 +34,7 @@ import org.apache.tinkerpop.gremlin.process.computer.ranking.pagerank.PageRankVe
import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
+import org.apache.tinkerpop.gremlin.spark.AbstractSparkTest;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkHadoopGraphProvider;
import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
@@ -50,10 +51,9 @@ import static org.junit.Assert.*;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-public class PersistedInputOutputRDDTest {
+public class PersistedInputOutputRDDTest extends AbstractSparkTest {
@Test
-
public void shouldNotPersistRDDAcrossJobs() throws Exception {
final String rddName = "target/test-output/" + UUID.randomUUID();
final Configuration configuration = new BaseConfiguration();
[17/17] incubator-tinkerpop git commit: Merge branch 'master' into
TINKERPOP3-1014
Posted by ok...@apache.org.
Merge branch 'master' into TINKERPOP3-1014
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/9af66047
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/9af66047
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/9af66047
Branch: refs/heads/TINKERPOP3-1014
Commit: 9af660476ec291fa12a499bb60349c8244167509
Parents: 83b93e0 1134ad2
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Thu Dec 3 13:31:03 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Dec 3 13:31:03 2015 -0700
----------------------------------------------------------------------
.travis.yml | 5 +-
CHANGELOG.asciidoc | 6 +
docs/src/reference/the-traversal.asciidoc | 33 +++--
.../gremlin/process/traversal/Order.java | 28 +++-
.../traversal/dsl/graph/GraphTraversal.java | 20 ++-
.../traversal/step/map/SelectColumnStep.java | 68 ----------
.../traversal/step/util/FunctionComparator.java | 56 ++++++++
.../tinkerpop/gremlin/structure/Column.java | 39 +++++-
.../step/branch/GroovyLocalTest.groovy | 2 +-
.../traversal/step/map/GroovyOrderTest.groovy | 8 +-
.../process/GroovyProcessComputerSuite.java | 2 +
.../process/GroovyProcessStandardSuite.java | 2 +
.../groovy/engine/GremlinExecutorTest.java | 8 +-
.../jsr223/GremlinGroovyScriptEngineTest.java | 52 ++++---
.../BulkLoaderVertexProgramTest.java | 2 +-
.../process/traversal/step/map/OrderTest.java | 11 +-
.../structure/hdfs/HadoopEdgeIterator.java | 4 -
.../structure/hdfs/HadoopElementIterator.java | 35 ++---
.../structure/hdfs/HadoopVertexIterator.java | 4 -
.../HadoopGraphStructureStandardTest.java | 35 +++++
.../io/HadoopGraphStructureStandardTest.java | 35 -----
pom.xml | 10 +-
.../process/computer/SparkContextHelper.java | 39 ++++++
.../process/computer/SparkGraphComputer.java | 15 +-
.../computer/payload/MessagePayload.java | 5 +-
.../computer/payload/ViewIncomingPayload.java | 6 +-
.../computer/payload/ViewOutgoingPayload.java | 12 +-
.../spark/structure/io/InputRDDFormat.java | 136 +++++++++++++++++++
.../spark/structure/io/gryo/GryoSerializer.java | 6 +
.../io/gryo/WrappedArraySerializer.java | 46 +++++++
.../gremlin/spark/AbstractSparkTest.java | 43 ++++++
.../process/computer/LocalPropertyTest.java | 8 +-
.../computer/SparkHadoopGraphProvider.java | 22 ++-
.../spark/structure/io/InputOutputRDDTest.java | 7 +-
.../spark/structure/io/InputRDDTest.java | 28 +++-
.../spark/structure/io/OutputRDDTest.java | 7 +-
.../io/PersistedInputOutputRDDTest.java | 14 +-
.../spark/structure/io/ToyGraphInputRDD.java | 73 ++++++++++
.../structure/TinkerGraphPlayTest.java | 7 +-
39 files changed, 698 insertions(+), 241 deletions(-)
----------------------------------------------------------------------
[03/17] incubator-tinkerpop git commit: Merge remote-tracking branch
'origin/TINKERPOP3-1006'
Posted by ok...@apache.org.
Merge remote-tracking branch 'origin/TINKERPOP3-1006'
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/0e43bdd3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/0e43bdd3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/0e43bdd3
Branch: refs/heads/TINKERPOP3-1014
Commit: 0e43bdd3ef8c65219b568aadcffd26be2e969c38
Parents: b70f3c7 8720165
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Thu Dec 3 14:24:03 2015 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Dec 3 14:24:03 2015 -0500
----------------------------------------------------------------------
.../groovy/engine/GremlinExecutorTest.java | 8 +--
.../jsr223/GremlinGroovyScriptEngineTest.java | 52 ++++++++++++--------
2 files changed, 35 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
[02/17] incubator-tinkerpop git commit: Drop build of javadocs and
asciidoc dryrun from travis
Posted by ok...@apache.org.
Drop build of javadocs and asciidoc dryrun from travis
Trying to shorten up the logs in travis a bit.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/b70f3c74
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/b70f3c74
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/b70f3c74
Branch: refs/heads/TINKERPOP3-1014
Commit: b70f3c744ead6d1605aa4d2b88073d014ad4df13
Parents: cdc063a
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Thu Dec 3 09:44:22 2015 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Dec 3 09:44:22 2015 -0500
----------------------------------------------------------------------
.travis.yml | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b70f3c74/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 8619a81..a8f56e5 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,6 +9,4 @@ addons:
packages:
- oracle-java8-installer
script:
- - "mvn clean install"
- - "bin/process-docs.sh --dryRun"
- - "mvn process-resources -Djavadoc"
\ No newline at end of file
+ - "mvn clean install"
\ No newline at end of file
[07/17] incubator-tinkerpop git commit: this is a breaking push. need
to get Groovy GremlinLoader happy about comparator. Will get back to this.
Posted by ok...@apache.org.
this is a breaking push. need to get Groovy GremlinLoader happy about comparator. Will get back to this.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/5b9de833
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/5b9de833
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/5b9de833
Branch: refs/heads/TINKERPOP3-1014
Commit: 5b9de8333286cdf32d00b0f706b0f6f21600f30e
Parents: b7c6041
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon Nov 30 12:45:53 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Dec 3 12:47:52 2015 -0700
----------------------------------------------------------------------
.../java/org/apache/tinkerpop/gremlin/structure/Column.java | 2 +-
.../process/traversal/step/branch/GroovyLocalTest.groovy | 2 +-
.../process/traversal/step/map/GroovyOrderTest.groovy | 8 ++++----
.../gremlin/process/traversal/step/map/OrderTest.java | 8 ++++----
4 files changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/5b9de833/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Column.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Column.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Column.java
index fca90d7..38921c1 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Column.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Column.java
@@ -25,7 +25,7 @@ import java.util.Map;
import java.util.function.Function;
/**
- * Column references a particular type of column in a complex data structure such as a {@link java.util.Map} or a {@link org.apache.tinkerpop.gremlin.process.traversal.Path}.
+ * Column references a particular type of column in a complex data structure such as a {@link java.util.Map}, a {@link java.util.Map.Entry}, or a {@link org.apache.tinkerpop.gremlin.process.traversal.Path}.
*
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/5b9de833/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/branch/GroovyLocalTest.groovy
----------------------------------------------------------------------
diff --git a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/branch/GroovyLocalTest.groovy b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/branch/GroovyLocalTest.groovy
index 7ed2673..fe8d629 100644
--- a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/branch/GroovyLocalTest.groovy
+++ b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/branch/GroovyLocalTest.groovy
@@ -32,7 +32,7 @@ public abstract class GroovyLocalTest {
@Override
public Traversal<Vertex, String> get_g_V_localXpropertiesXlocationX_order_byXvalueX_limitX2XX_value() {
- TraversalScriptHelper.compute("g.V.local(__.properties('location').order.by(T.value,Order.incr).limit(2)).value", g);
+ TraversalScriptHelper.compute("g.V.local(__.properties('location').order.by(value,incr).limit(2)).value", g);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/5b9de833/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyOrderTest.groovy
----------------------------------------------------------------------
diff --git a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyOrderTest.groovy b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyOrderTest.groovy
index e9463bc..002a4e1 100644
--- a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyOrderTest.groovy
+++ b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyOrderTest.groovy
@@ -41,12 +41,12 @@ public abstract class GroovyOrderTest {
@Override
public Traversal<Vertex, String> get_g_V_order_byXname_incrX_name() {
- TraversalScriptHelper.compute("g.V.order.by('name', Order.incr).name", g)
+ TraversalScriptHelper.compute("g.V.order.by('name', incr).name", g)
}
@Override
public Traversal<Vertex, String> get_g_V_order_byXnameX_name() {
- TraversalScriptHelper.compute("g.V.order.by('name', Order.incr).name", g)
+ TraversalScriptHelper.compute("g.V.order.by('name', incr).name", g)
}
@Override
@@ -65,7 +65,7 @@ public abstract class GroovyOrderTest {
}
@Override
- public Traversal<Vertex, Map<Integer, Integer>> get_g_VX1X_hasXlabel_personX_mapXmapXint_ageXX_orderXlocalX_byXvalueDecrX_byXkeyIncrX(
+ public Traversal<Vertex, Map<Integer, Integer>> get_g_VX1X_hasXlabel_personX_mapXmapXint_ageXX_orderXlocalX_byXvalues_decrX_byXkeys_incrX(
final Object v1Id) {
TraversalScriptHelper.compute("""g.V(v1Id).map {
final Map map = [:];
@@ -74,7 +74,7 @@ public abstract class GroovyOrderTest {
map[3] = it.age * 3;
map[4] = it.age;
return map;
- }.order(local).by(valueDecr).by(keyIncr)""", g, "v1Id", v1Id);
+ }.order(local).by(values,decr).by(keys,incr)""", g, "v1Id", v1Id);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/5b9de833/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderTest.java
index 3f08d60..4475b7c 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderTest.java
@@ -65,7 +65,7 @@ public abstract class OrderTest extends AbstractGremlinProcessTest {
public abstract Traversal<Vertex, Map<String, Vertex>> get_g_V_asXaX_outXcreatedX_asXbX_order_byXshuffleX_selectXa_bX();
- public abstract Traversal<Vertex, Map<Integer, Integer>> get_g_VX1X_hasXlabel_personX_mapXmapXint_ageXX_orderXlocalX_byXvalueDecrX_byXkeyIncrX(final Object v1Id);
+ public abstract Traversal<Vertex, Map<Integer, Integer>> get_g_VX1X_hasXlabel_personX_mapXmapXint_ageXX_orderXlocalX_byXvalues_decrX_byXkeys_incrX(final Object v1Id);
public abstract Traversal<Vertex, Vertex> get_g_V_order_byXoutE_count__decrX();
@@ -196,8 +196,8 @@ public abstract class OrderTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(MODERN)
- public void g_VX1X_hasXlabel_personX_mapXmapXint_ageXX_orderXlocalX_byXvalueDecrX_byXkeyIncrX() {
- final Traversal<Vertex, Map<Integer, Integer>> traversal = get_g_VX1X_hasXlabel_personX_mapXmapXint_ageXX_orderXlocalX_byXvalueDecrX_byXkeyIncrX(convertToVertexId("marko"));
+ public void g_VX1X_hasXlabel_personX_mapXmapXint_ageXX_orderXlocalX_byXvalues_decrX_byXkeys_incrX() {
+ final Traversal<Vertex, Map<Integer, Integer>> traversal = get_g_VX1X_hasXlabel_personX_mapXmapXint_ageXX_orderXlocalX_byXvalues_decrX_byXkeys_incrX(convertToVertexId("marko"));
printTraversalForm(traversal);
final Map<Integer, Integer> map = traversal.next();
assertFalse(traversal.hasNext());
@@ -356,7 +356,7 @@ public abstract class OrderTest extends AbstractGremlinProcessTest {
}
@Override
- public Traversal<Vertex, Map<Integer, Integer>> get_g_VX1X_hasXlabel_personX_mapXmapXint_ageXX_orderXlocalX_byXvalueDecrX_byXkeyIncrX(final Object v1Id) {
+ public Traversal<Vertex, Map<Integer, Integer>> get_g_VX1X_hasXlabel_personX_mapXmapXint_ageXX_orderXlocalX_byXvalues_decrX_byXkeys_incrX(final Object v1Id) {
return g.V(v1Id).map(v -> {
final Map<Integer, Integer> map = new HashMap<>();
map.put(1, (int) v.get().value("age"));