You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by sp...@apache.org on 2015/12/04 15:21:05 UTC
[01/31] incubator-tinkerpop git commit: Added Traverser.getTags().
This gives us more flexbility in how we handle branch-history. This improves
MatchStep, future IntersectStep, future SymmetricDifferenceStep,
and will allow us to do traversal rewriting i
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/TINKERPOP3-860 b503e4155 -> b27a30b2e
Added Traverser.getTags(). This gives us more flexbility in how we handle branch-history. This improves MatchStep, future IntersectStep, future SymmetricDifferenceStep, and will allow us to do traversal rewriting in MatchStep (TINKERPOP3-736). No more awkward hack of traverser.path().extend(step.getId())... the stepID is NOT a path label, its a tag.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/e6a1141a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/e6a1141a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/e6a1141a
Branch: refs/heads/TINKERPOP3-860
Commit: e6a1141a0515f8adc7e51adb2a84495349b06c6a
Parents: fdd853b
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Dec 1 13:16:26 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue Dec 1 13:16:26 2015 -0700
----------------------------------------------------------------------
.../gremlin/process/traversal/Traverser.java | 9 ++++
.../process/traversal/step/map/MatchStep.java | 56 +++++++++++++-------
.../traversal/step/util/ComputerAwareStep.java | 2 +-
.../traversal/traverser/O_Traverser.java | 37 +++++++++++++
.../traverser/util/EmptyTraverser.java | 6 +++
.../traversal/step/map/MatchStepTest.java | 15 ++++--
6 files changed, 99 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/e6a1141a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/Traverser.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/Traverser.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/Traverser.java
index 06925a8..da43fdf 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/Traverser.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/Traverser.java
@@ -272,5 +272,14 @@ public interface Traverser<T> extends Serializable, Comparable<Traverser<T>>, Cl
* @return the traversal sideEffects of the traverser
*/
public TraversalSideEffects getSideEffects();
+
+ /**
+ * Get the tags associated with the traverser.
+ * Tags are used to categorize historic behavior of a traverser.
+ * The returned set is mutable.
+ *
+ * @return the set of tags associated with the traverser.
+ */
+ public Set<String> getTags();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/e6a1141a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MatchStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MatchStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MatchStep.java
index 38da656..5452c17 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MatchStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MatchStep.java
@@ -206,7 +206,7 @@ public final class MatchStep<S, E> extends ComputerAwareStep<S, Map<String, E>>
}
public MatchAlgorithm getMatchAlgorithm() {
- if(null == this.matchAlgorithm)
+ if (null == this.matchAlgorithm)
this.initializeMatchAlgorithm(this.traverserStepIdAndLabelsSetByChild ? TraversalEngine.Type.COMPUTER : TraversalEngine.Type.STANDARD);
return this.matchAlgorithm;
}
@@ -236,23 +236,23 @@ public final class MatchStep<S, E> extends ComputerAwareStep<S, Map<String, E>>
private boolean isDuplicate(final Traverser<S> traverser) {
if (null == this.dedups)
return false;
+ final Path path = traverser.path();
for (final String label : this.dedupLabels) {
- if (!traverser.path().hasLabel(label))
+ if (!path.hasLabel(label))
return false;
}
final List<Object> objects = new ArrayList<>(this.dedupLabels.size());
for (final String label : this.dedupLabels) {
- objects.add(traverser.path().get(Pop.last, label));
+ objects.add(path.get(Pop.last, label));
}
return this.dedups.contains(objects);
}
- private boolean hasMatched(final ConnectiveStep.Connective connective, final Traverser<S> traverser) {
- final Path path = traverser.path();
+ private boolean hasMatched(final ConnectiveStep.Connective connective, final Traverser.Admin<S> traverser) {
int counter = 0;
boolean matched = false;
for (final Traversal.Admin<Object, Object> matchTraversal : this.matchTraversals) {
- if (path.hasLabel(matchTraversal.getStartStep().getId())) {
+ if (traverser.getTags().contains(matchTraversal.getStartStep().getId())) {
if (connective == ConnectiveStep.Connective.OR) {
matched = true;
break;
@@ -263,9 +263,10 @@ public final class MatchStep<S, E> extends ComputerAwareStep<S, Map<String, E>>
if (!matched)
matched = this.matchTraversals.size() == counter;
if (matched && this.dedupLabels != null) {
+ final Path path = traverser.path();
final List<Object> objects = new ArrayList<>(this.dedupLabels.size());
for (final String label : this.dedupLabels) {
- objects.add(traverser.path().get(Pop.last, label));
+ objects.add(path.get(Pop.last, label));
}
this.dedups.add(objects);
}
@@ -292,6 +293,14 @@ public final class MatchStep<S, E> extends ComputerAwareStep<S, Map<String, E>>
this.matchAlgorithm.initialize(traversalEngineType, this.matchTraversals);
}
+ private boolean hasPathLabel(final Path path, final Set<String> labels) {
+ for (final String label : labels) {
+ if (path.hasLabel(label))
+ return true;
+ }
+ return false;
+ }
+
@Override
protected Iterator<Traverser<Map<String, E>>> standardAlgorithm() throws NoSuchElementException {
while (true) {
@@ -309,10 +318,9 @@ public final class MatchStep<S, E> extends ComputerAwareStep<S, Map<String, E>>
}
if (null == traverser) {
traverser = this.starts.next();
- final Path path = traverser.path();
- if (!this.matchStartLabels.stream().filter(path::hasLabel).findAny().isPresent())
+ if (!this.hasPathLabel(traverser.path(), this.matchStartLabels))
traverser.addLabels(Collections.singleton(this.computedStartLabel)); // if the traverser doesn't have a legal start, then provide it the pre-computed one
- traverser.addLabels(Collections.singleton(this.getId())); // so the traverser never returns to this branch ever again
+ traverser.getTags().add(this.getId()); // so the traverser never returns to this branch ever again
}
///
if (!this.isDuplicate(traverser)) {
@@ -320,10 +328,14 @@ public final class MatchStep<S, E> extends ComputerAwareStep<S, Map<String, E>>
return IteratorUtils.of(traverser.split(this.getBindings(traverser), this));
if (this.connective == ConnectiveStep.Connective.AND) {
- this.getMatchAlgorithm().apply(traverser).addStart(traverser); // determine which sub-pattern the traverser should try next
+ final Traversal.Admin<Object, Object> matchTraversal = this.getMatchAlgorithm().apply(traverser);
+ traverser.getTags().add(matchTraversal.getStartStep().getId());
+ matchTraversal.addStart(traverser); // determine which sub-pattern the traverser should try next
} else { // OR
for (final Traversal.Admin<?, ?> matchTraversal : this.matchTraversals) {
- matchTraversal.addStart(traverser.split());
+ final Traverser.Admin split = traverser.split();
+ split.getTags().add(matchTraversal.getStartStep().getId());
+ matchTraversal.addStart(split);
}
}
}
@@ -338,11 +350,9 @@ public final class MatchStep<S, E> extends ComputerAwareStep<S, Map<String, E>>
this.initializeMatchAlgorithm(TraversalEngine.Type.COMPUTER);
}
final Traverser.Admin traverser = this.starts.next();
- final Path path = traverser.path();
- if (!this.matchStartLabels.stream().filter(path::hasLabel).findAny().isPresent())
+ if (!this.hasPathLabel(traverser.path(), this.matchStartLabels))
traverser.addLabels(Collections.singleton(this.computedStartLabel)); // if the traverser doesn't have a legal start, then provide it the pre-computed one
- if (!path.hasLabel(this.getId()))
- traverser.addLabels(Collections.singleton(this.getId())); // so the traverser never returns to this branch ever again
+ traverser.getTags().add(this.getId()); // so the traverser never returns to this branch ever again
///
if (!this.isDuplicate(traverser)) {
if (hasMatched(this.connective, traverser)) {
@@ -352,15 +362,17 @@ public final class MatchStep<S, E> extends ComputerAwareStep<S, Map<String, E>>
}
if (this.connective == ConnectiveStep.Connective.AND) {
final Traversal.Admin<Object, Object> matchTraversal = this.getMatchAlgorithm().apply(traverser); // determine which sub-pattern the traverser should try next
+ traverser.getTags().add(matchTraversal.getStartStep().getId());
traverser.setStepId(matchTraversal.getStartStep().getId()); // go down the traversal match sub-pattern
return IteratorUtils.of(traverser);
} else { // OR
final List<Traverser<Map<String, E>>> traversers = new ArrayList<>(this.matchTraversals.size());
- this.matchTraversals.forEach(matchTraversal -> {
+ for (final Traversal.Admin<?, ?> matchTraversal : this.matchTraversals) {
final Traverser.Admin split = traverser.split();
+ split.getTags().add(matchTraversal.getStartStep().getId());
split.setStepId(matchTraversal.getStartStep().getId());
traversers.add(split);
- });
+ }
return traversers.iterator();
}
}
@@ -503,7 +515,11 @@ public final class MatchStep<S, E> extends ComputerAwareStep<S, Map<String, E>>
}
public static boolean hasStartLabels(final Traverser.Admin<Object> traverser, final Traversal.Admin<Object, Object> traversal) {
- return !Helper.getStartLabels(traversal).stream().filter(label -> !traverser.path().hasLabel(label)).findAny().isPresent();
+ for (final String label : Helper.getStartLabels(traversal)) {
+ if (!traverser.path().hasLabel(label))
+ return false;
+ }
+ return true;
}
public static boolean hasEndLabel(final Traverser.Admin<Object> traverser, final Traversal.Admin<Object, Object> traversal) {
@@ -512,7 +528,7 @@ public final class MatchStep<S, E> extends ComputerAwareStep<S, Map<String, E>>
}
public static boolean hasExecutedTraversal(final Traverser.Admin<Object> traverser, final Traversal.Admin<Object, Object> traversal) {
- return traverser.path().hasLabel(traversal.getStartStep().getId());
+ return traverser.getTags().contains(traversal.getStartStep().getId());
}
public static TraversalType getTraversalType(final Traversal.Admin<Object, Object> traversal) {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/e6a1141a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/ComputerAwareStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/ComputerAwareStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/ComputerAwareStep.java
index 2c7ecc5..419205c 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/ComputerAwareStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/ComputerAwareStep.java
@@ -77,7 +77,7 @@ public abstract class ComputerAwareStep<S, E> extends AbstractStep<S, E> impleme
final Traverser.Admin<S> start = this.starts.next();
if (this.traverserStepIdAndLabelsSetByChild) {
start.setStepId(((ComputerAwareStep<?, ?>) this.getTraversal().getParent()).getNextStep().getId());
- start.addLabels(((ComputerAwareStep<?, ?>) this.getTraversal().getParent()).getLabels());
+ start.path().extend(((ComputerAwareStep<?, ?>) this.getTraversal().getParent()).getLabels());
}
return start;
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/e6a1141a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/traverser/O_Traverser.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/traverser/O_Traverser.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/traverser/O_Traverser.java
index 4fc3f5d..44e0f6e 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/traverser/O_Traverser.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/traverser/O_Traverser.java
@@ -18,13 +18,20 @@
*/
package org.apache.tinkerpop.gremlin.process.traversal.traverser;
+import org.apache.tinkerpop.gremlin.process.traversal.Step;
+import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.AbstractTraverser;
+import java.util.HashSet;
+import java.util.Set;
+
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
public class O_Traverser<T> extends AbstractTraverser<T> {
+ protected Set<String> tags = null;
+
protected O_Traverser() {
}
@@ -32,4 +39,34 @@ public class O_Traverser<T> extends AbstractTraverser<T> {
super(t);
}
+
+ public Set<String> getTags() {
+ if (null == this.tags) this.tags = new HashSet<>();
+ return this.tags;
+ }
+
+ @Override
+ public <R> Admin<R> split(final R r, final Step<T, R> step) {
+ final O_Traverser<R> clone = (O_Traverser<R>) super.split(r, step);
+ if (null != this.tags)
+ clone.tags = new HashSet<>(this.tags);
+ return clone;
+ }
+
+ @Override
+ public Admin<T> split() {
+ final O_Traverser<T> clone = (O_Traverser<T>) super.split();
+ if (null != this.tags)
+ clone.tags = new HashSet<>(this.tags);
+ return clone;
+ }
+
+ @Override
+ public void merge(final Traverser.Admin<?> other) {
+ super.merge(other);
+ if (!other.getTags().isEmpty()) {
+ if (this.tags == null) this.tags = new HashSet<>();
+ this.tags.addAll(other.getTags());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/e6a1141a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/traverser/util/EmptyTraverser.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/traverser/util/EmptyTraverser.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/traverser/util/EmptyTraverser.java
index a099505..da391f5 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/traverser/util/EmptyTraverser.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/traverser/util/EmptyTraverser.java
@@ -25,6 +25,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.EmptyPath;
import org.apache.tinkerpop.gremlin.structure.util.Attachable;
+import java.util.Collections;
import java.util.Set;
import java.util.function.Function;
@@ -144,6 +145,11 @@ public final class EmptyTraverser<T> implements Traverser<T>, Traverser.Admin<T>
}
@Override
+ public Set<String> getTags() {
+ return Collections.emptySet();
+ }
+
+ @Override
public int hashCode() {
return 380473707;
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/e6a1141a/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MatchStepTest.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MatchStepTest.java b/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MatchStepTest.java
index 4f7c231..7360e24 100644
--- a/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MatchStepTest.java
+++ b/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MatchStepTest.java
@@ -365,23 +365,28 @@ public class MatchStepTest extends StepTest {
traverser.addLabels(Collections.singleton("a"));
assertEquals(firstPattern, countMatchAlgorithm.apply(traverser));
traverser = traverser.split(1, EmptyStep.instance());
- traverser.addLabels(new HashSet<>(Arrays.asList("b", firstPattern.getStartStep().getId())));
+ traverser.getTags().add(firstPattern.getStartStep().getId());
+ traverser.addLabels(Collections.singleton("b"));
//
assertEquals(secondPattern, countMatchAlgorithm.apply(traverser));
traverser = traverser.split(1, EmptyStep.instance());
- traverser.addLabels(new HashSet<>(Arrays.asList("c", secondPattern.getStartStep().getId())));
+ traverser.getTags().add(secondPattern.getStartStep().getId());
+ traverser.addLabels(Collections.singleton("c"));
//
assertEquals(fifthPattern, countMatchAlgorithm.apply(traverser));
traverser = traverser.split(1, EmptyStep.instance());
- traverser.addLabels(new HashSet<>(Arrays.asList("f", fifthPattern.getStartStep().getId())));
+ traverser.getTags().add(fifthPattern.getStartStep().getId());
+ traverser.addLabels(Collections.singleton("f"));
//
assertEquals(forthPattern, countMatchAlgorithm.apply(traverser));
traverser = traverser.split(1, EmptyStep.instance());
- traverser.addLabels(new HashSet<>(Arrays.asList("e", forthPattern.getStartStep().getId())));
+ traverser.getTags().add(forthPattern.getStartStep().getId());
+ traverser.addLabels(Collections.singleton("e"));
//
assertEquals(thirdPattern, countMatchAlgorithm.apply(traverser));
traverser = traverser.split(1, EmptyStep.instance());
- traverser.addLabels(new HashSet<>(Arrays.asList("d", thirdPattern.getStartStep().getId())));
+ traverser.getTags().add(thirdPattern.getStartStep().getId());
+ traverser.addLabels(Collections.singleton("d"));
}
@Test
[07/31] incubator-tinkerpop git commit: Drop build of javadocs and
asciidoc dryrun from travis
Posted by sp...@apache.org.
Drop build of javadocs and asciidoc dryrun from travis
Trying to shorten up the logs in travis a bit.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/b70f3c74
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/b70f3c74
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/b70f3c74
Branch: refs/heads/TINKERPOP3-860
Commit: b70f3c744ead6d1605aa4d2b88073d014ad4df13
Parents: cdc063a
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Thu Dec 3 09:44:22 2015 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Dec 3 09:44:22 2015 -0500
----------------------------------------------------------------------
.travis.yml | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b70f3c74/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 8619a81..a8f56e5 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,6 +9,4 @@ addons:
packages:
- oracle-java8-installer
script:
- - "mvn clean install"
- - "bin/process-docs.sh --dryRun"
- - "mvn process-resources -Djavadoc"
\ No newline at end of file
+ - "mvn clean install"
\ No newline at end of file
[10/31] incubator-tinkerpop git commit: some organization and clean
up. Stuff is lookin SOLID. Time to run full integration tests.
Posted by sp...@apache.org.
some organization and clean up. Stuff is lookin SOLID. Time to run full integration tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/26dcf3c9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/26dcf3c9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/26dcf3c9
Branch: refs/heads/TINKERPOP3-860
Commit: 26dcf3c943bf1328747cd78d6fd5a814dfe6a5da
Parents: ddbbec0
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed Dec 2 13:11:29 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Dec 3 12:47:52 2015 -0700
----------------------------------------------------------------------
.../process/computer/SparkContextHelper.java | 39 +++++++++++
.../process/computer/SparkGraphComputer.java | 15 ++---
.../spark/structure/io/InputRDDFormat.java | 8 +--
.../process/computer/LocalPropertyTest.java | 4 +-
.../computer/SparkHadoopGraphProvider.java | 38 +++--------
.../spark/structure/io/ClassicInputRDD.java | 39 -----------
.../spark/structure/io/GratefulInputRDD.java | 50 --------------
.../spark/structure/io/InputOutputRDDTest.java | 1 -
.../spark/structure/io/InputRDDTest.java | 1 -
.../spark/structure/io/ModernInputRDD.java | 41 ------------
.../spark/structure/io/OutputRDDTest.java | 1 -
.../io/PersistedInputOutputRDDTest.java | 10 +--
.../spark/structure/io/TheCrewInputRDD.java | 39 -----------
.../spark/structure/io/ToyGraphInputRDD.java | 69 ++++++++++++++++++++
14 files changed, 134 insertions(+), 221 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26dcf3c9/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkContextHelper.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkContextHelper.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkContextHelper.java
new file mode 100644
index 0000000..ed553c5
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkContextHelper.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.process.computer;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class SparkContextHelper {
+
+ private SparkContextHelper() {
+
+ }
+
+ public static void tryToCloseContext(final JavaSparkContext context, final Configuration configuration) {
+ if (context != null && !configuration.getBoolean(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, false))
+ context.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26dcf3c9/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
index e7566d5..c20f1b0 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
@@ -228,8 +228,7 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
finalMemory.setRuntime(System.currentTimeMillis() - startTime);
return new DefaultComputerResult(InputOutputHelper.getOutputGraph(apacheConfiguration, this.resultGraph, this.persist), finalMemory.asImmutable());
} finally {
- if (sparkContext != null && !apacheConfiguration.getBoolean(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, false))
- sparkContext.stop();
+ SparkContextHelper.tryToCloseContext(sparkContext, apacheConfiguration);
}
}, exec);
}
@@ -268,14 +267,14 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
* Execution rather than applying the entire configuration.
*/
final String[] validPropertyNames = {
- "spark.job.description",
- "spark.jobGroup.id",
- "spark.job.interruptOnCancel",
- "spark.scheduler.pool"
+ "spark.job.description",
+ "spark.jobGroup.id",
+ "spark.job.interruptOnCancel",
+ "spark.scheduler.pool"
};
- for (String propertyName: validPropertyNames){
- if (sparkConfiguration.contains(propertyName)){
+ for (String propertyName : validPropertyNames) {
+ if (sparkConfiguration.contains(propertyName)) {
String propertyValue = sparkConfiguration.get(propertyName);
this.logger.info("Setting Thread Local SparkContext Property - "
+ propertyName + " : " + propertyValue);
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26dcf3c9/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDFormat.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDFormat.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDFormat.java
index 3952c66..d1a198e 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDFormat.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDFormat.java
@@ -31,6 +31,7 @@ import org.apache.spark.api.java.JavaSparkContext;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
+import org.apache.tinkerpop.gremlin.spark.process.computer.SparkContextHelper;
import scala.Tuple2;
import java.io.IOException;
@@ -70,8 +71,8 @@ public final class InputRDDFormat extends InputFormat<NullWritable, VertexWritab
final SparkConf sparkConfiguration = new SparkConf();
sparkConfiguration.setAppName(UUID.randomUUID().toString());
hadoopConfiguration.forEach(entry -> sparkConfiguration.set(entry.getKey(), entry.getValue()));
- InputRDD inputRDD = (InputRDD) Class.forName(sparkConfiguration.get(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD)).newInstance();
- JavaSparkContext javaSparkContext = new JavaSparkContext(SparkContext.getOrCreate(sparkConfiguration));
+ final InputRDD inputRDD = (InputRDD) Class.forName(sparkConfiguration.get(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD)).newInstance();
+ final JavaSparkContext javaSparkContext = new JavaSparkContext(SparkContext.getOrCreate(sparkConfiguration));
final Iterator<Tuple2<Object, VertexWritable>> iterator = inputRDD.readGraphRDD(ConfUtil.makeApacheConfiguration(taskAttemptContext.getConfiguration()), javaSparkContext).toLocalIterator();
return new RecordReader<NullWritable, VertexWritable>() {
@Override
@@ -101,8 +102,7 @@ public final class InputRDDFormat extends InputFormat<NullWritable, VertexWritab
@Override
public void close() throws IOException {
- if (!hadoopConfiguration.getBoolean(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, false))
- javaSparkContext.close();
+ SparkContextHelper.tryToCloseContext(javaSparkContext, ConfUtil.makeApacheConfiguration(hadoopConfiguration));
}
};
} catch (final ClassNotFoundException | InstantiationException | IllegalAccessException e) {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26dcf3c9/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/LocalPropertyTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/LocalPropertyTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/LocalPropertyTest.java
index e0fe796..671bee8 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/LocalPropertyTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/LocalPropertyTest.java
@@ -25,7 +25,6 @@ import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.JavaSparkStatusTracker;
-import org.apache.spark.serializer.KryoSerializer;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
@@ -36,6 +35,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSo
import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
import org.apache.tinkerpop.gremlin.spark.structure.io.PersistedInputRDD;
import org.apache.tinkerpop.gremlin.spark.structure.io.PersistedOutputRDD;
+import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
import org.junit.Test;
@@ -53,7 +53,7 @@ public class LocalPropertyTest {
final String rddName = "target/test-output/" + UUID.randomUUID();
final Configuration configuration = new BaseConfiguration();
configuration.setProperty("spark.master", "local[4]");
- configuration.setProperty("spark.serializer", KryoSerializer.class.getCanonicalName());
+ configuration.setProperty("spark.serializer", GryoSerializer.class.getCanonicalName());
configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, GryoInputFormat.class.getCanonicalName());
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26dcf3c9/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
index 2328176..2cfeea3 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
@@ -24,11 +24,9 @@ import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.HadoopGraphProvider;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
-import org.apache.tinkerpop.gremlin.spark.structure.io.ClassicInputRDD;
-import org.apache.tinkerpop.gremlin.spark.structure.io.GratefulInputRDD;
import org.apache.tinkerpop.gremlin.spark.structure.io.InputRDDFormat;
-import org.apache.tinkerpop.gremlin.spark.structure.io.ModernInputRDD;
-import org.apache.tinkerpop.gremlin.spark.structure.io.TheCrewInputRDD;
+import org.apache.tinkerpop.gremlin.spark.structure.io.ToyGraphInputRDD;
+import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
import org.apache.tinkerpop.gremlin.structure.Graph;
import java.util.Map;
@@ -46,36 +44,16 @@ public final class SparkHadoopGraphProvider extends HadoopGraphProvider {
public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
final Map<String, Object> config = super.getBaseConfiguration(graphName, test, testMethodName, loadGraphWith);
if (null != loadGraphWith) {
- if (loadGraphWith.equals(LoadGraphWith.GraphData.MODERN)) {
- if (RANDOM.nextBoolean()) {
- config.remove(Constants.GREMLIN_HADOOP_INPUT_LOCATION);
- config.put(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, ModernInputRDD.class.getCanonicalName());
- config.put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName());
- }
- } else if (loadGraphWith.equals(LoadGraphWith.GraphData.CREW)) {
- if (RANDOM.nextBoolean()) {
- config.remove(Constants.GREMLIN_HADOOP_INPUT_LOCATION);
- config.put(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, TheCrewInputRDD.class.getCanonicalName());
- config.put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName());
- }
- } else if (loadGraphWith.equals(LoadGraphWith.GraphData.CLASSIC)) {
- if (RANDOM.nextBoolean()) {
- config.remove(Constants.GREMLIN_HADOOP_INPUT_LOCATION);
- config.put(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, ClassicInputRDD.class.getCanonicalName());
- config.put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName());
- }
- } else if (loadGraphWith.equals(LoadGraphWith.GraphData.GRATEFUL)) {
- if (RANDOM.nextBoolean()) {
- config.remove(Constants.GREMLIN_HADOOP_INPUT_LOCATION);
- config.put(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, GratefulInputRDD.class.getCanonicalName());
- config.put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName());
- }
+ if (RANDOM.nextBoolean()) {
+ config.remove(Constants.GREMLIN_HADOOP_INPUT_LOCATION);
+ config.put(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, ToyGraphInputRDD.class.getCanonicalName());
+ config.put(ToyGraphInputRDD.GREMLIN_SPARK_TOY_GRAPH, loadGraphWith.toString());
+ config.put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName());
}
}
/// spark configuration
config.put("spark.master", "local[4]");
- //config.put("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
- config.put("spark.serializer", "org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer");
+ config.put("spark.serializer", GryoSerializer.class.getCanonicalName());
config.put("spark.kryo.registrationRequired", true);
return config;
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26dcf3c9/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ClassicInputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ClassicInputRDD.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ClassicInputRDD.java
deleted file mode 100644
index 4512b61..0000000
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ClassicInputRDD.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.spark.structure.io;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory;
-import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
-import scala.Tuple2;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class ClassicInputRDD implements InputRDD {
-
- @Override
- public JavaPairRDD<Object, VertexWritable> readGraphRDD(final Configuration configuration, final JavaSparkContext sparkContext) {
- return sparkContext.parallelize(IteratorUtils.list(IteratorUtils.map(TinkerFactory.createClassic().vertices(), VertexWritable::new))).mapToPair(vertex -> new Tuple2<>(vertex.get().id(), vertex));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26dcf3c9/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/GratefulInputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/GratefulInputRDD.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/GratefulInputRDD.java
deleted file mode 100644
index 396aa75..0000000
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/GratefulInputRDD.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.spark.structure.io;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess;
-import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
-import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
-import scala.Tuple2;
-
-import java.io.IOException;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GratefulInputRDD implements InputRDD {
-
- @Override
- public JavaPairRDD<Object, VertexWritable> readGraphRDD(final Configuration configuration, final JavaSparkContext sparkContext) {
- try {
- final Graph graph = TinkerGraph.open();
- graph.io(GryoIo.build()).readGraph(GryoResourceAccess.class.getResource("grateful-dead.kryo").getFile());
- return sparkContext.parallelize(IteratorUtils.list(IteratorUtils.map(graph.vertices(), VertexWritable::new))).mapToPair(vertex -> new Tuple2<>(vertex.get().id(), vertex));
- } catch (final IOException e) {
- throw new IllegalStateException(e.getMessage(), e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26dcf3c9/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputRDDTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputRDDTest.java
index ea62114..50a43bc 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputRDDTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputRDDTest.java
@@ -20,7 +20,6 @@ package org.apache.tinkerpop.gremlin.spark.structure.io;
import org.apache.commons.configuration.BaseConfiguration;
import org.apache.commons.configuration.Configuration;
-import org.apache.spark.serializer.KryoSerializer;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26dcf3c9/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
index 2cbfd66..b64139c 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
@@ -20,7 +20,6 @@ package org.apache.tinkerpop.gremlin.spark.structure.io;
import org.apache.commons.configuration.BaseConfiguration;
import org.apache.commons.configuration.Configuration;
-import org.apache.spark.serializer.KryoSerializer;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26dcf3c9/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ModernInputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ModernInputRDD.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ModernInputRDD.java
deleted file mode 100644
index 849e3e6..0000000
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ModernInputRDD.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.spark.structure.io;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory;
-import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
-import scala.Tuple2;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class ModernInputRDD implements InputRDD {
-
- @Override
- public JavaPairRDD<Object, VertexWritable> readGraphRDD(final Configuration configuration, final JavaSparkContext sparkContext) {
- return sparkContext.
- parallelize(IteratorUtils.list(IteratorUtils.map(TinkerFactory.createModern().vertices(), VertexWritable::new))).
- mapToPair(vertex -> new Tuple2<>(vertex.get().id(), vertex));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26dcf3c9/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java
index f9b6f39..60790e7 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java
@@ -20,7 +20,6 @@ package org.apache.tinkerpop.gremlin.spark.structure.io;
import org.apache.commons.configuration.BaseConfiguration;
import org.apache.commons.configuration.Configuration;
-import org.apache.spark.serializer.KryoSerializer;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26dcf3c9/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java
index 6aeb864..1de2b47 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java
@@ -24,7 +24,6 @@ import org.apache.commons.configuration.Configuration;
import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.serializer.KryoSerializer;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
@@ -37,6 +36,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSo
import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkHadoopGraphProvider;
+import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.io.IoCore;
import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
@@ -58,7 +58,7 @@ public class PersistedInputOutputRDDTest {
final String rddName = "target/test-output/" + UUID.randomUUID();
final Configuration configuration = new BaseConfiguration();
configuration.setProperty("spark.master", "local[4]");
- configuration.setProperty("spark.serializer", KryoSerializer.class.getCanonicalName());
+ configuration.setProperty("spark.serializer", GryoSerializer.class.getCanonicalName());
configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, GryoInputFormat.class.getCanonicalName());
@@ -87,7 +87,7 @@ public class PersistedInputOutputRDDTest {
final String rddName = "target/test-output/" + UUID.randomUUID();
final Configuration configuration = new BaseConfiguration();
configuration.setProperty("spark.master", "local[4]");
- configuration.setProperty("spark.serializer", KryoSerializer.class.getCanonicalName());
+ configuration.setProperty("spark.serializer", GryoSerializer.class.getCanonicalName());
configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, GryoInputFormat.class.getCanonicalName());
@@ -129,7 +129,7 @@ public class PersistedInputOutputRDDTest {
final String rddName = "target/test-output/" + UUID.randomUUID().toString();
final Configuration readConfiguration = new BaseConfiguration();
readConfiguration.setProperty("spark.master", "local[4]");
- readConfiguration.setProperty("spark.serializer", KryoSerializer.class.getCanonicalName());
+ readConfiguration.setProperty("spark.serializer", GryoSerializer.class.getCanonicalName());
readConfiguration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
readConfiguration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, GryoInputFormat.class.getCanonicalName());
readConfiguration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
@@ -176,7 +176,7 @@ public class PersistedInputOutputRDDTest {
final String rddName = "target/test-output/" + UUID.randomUUID().toString();
final Configuration readConfiguration = new BaseConfiguration();
readConfiguration.setProperty("spark.master", "local[4]");
- readConfiguration.setProperty("spark.serializer", KryoSerializer.class.getCanonicalName());
+ readConfiguration.setProperty("spark.serializer", GryoSerializer.class.getCanonicalName());
readConfiguration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
readConfiguration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, GryoInputFormat.class.getCanonicalName());
readConfiguration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26dcf3c9/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/TheCrewInputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/TheCrewInputRDD.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/TheCrewInputRDD.java
deleted file mode 100644
index ff5a274..0000000
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/TheCrewInputRDD.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.spark.structure.io;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory;
-import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
-import scala.Tuple2;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class TheCrewInputRDD implements InputRDD {
-
- @Override
- public JavaPairRDD<Object, VertexWritable> readGraphRDD(final Configuration configuration, final JavaSparkContext sparkContext) {
- return sparkContext.parallelize(IteratorUtils.list(IteratorUtils.map(TinkerFactory.createTheCrew().vertices(), VertexWritable::new))).mapToPair(vertex -> new Tuple2<>(vertex.get().id(), vertex));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26dcf3c9/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
new file mode 100644
index 0000000..ed97c04
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.structure.io;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.tinkerpop.gremlin.LoadGraphWith;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+import scala.Tuple2;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class ToyGraphInputRDD implements InputRDD {
+
+ public static final String GREMLIN_SPARK_TOY_GRAPH = "gremlin.spark.toyGraph";
+
+ @Override
+ public JavaPairRDD<Object, VertexWritable> readGraphRDD(final Configuration configuration, final JavaSparkContext sparkContext) {
+ final List<Vertex> vertices;
+ if (configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH).equals(LoadGraphWith.GraphData.MODERN.toString()))
+ vertices = IteratorUtils.list(TinkerFactory.createModern().vertices());
+ else if (configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH).equals(LoadGraphWith.GraphData.CLASSIC.toString()))
+ vertices = IteratorUtils.list(TinkerFactory.createClassic().vertices());
+ else if (configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH).equals(LoadGraphWith.GraphData.CREW.toString()))
+ vertices = IteratorUtils.list(TinkerFactory.createTheCrew().vertices());
+ else if (configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH).equals(LoadGraphWith.GraphData.GRATEFUL.toString())) {
+ try {
+ final Graph graph = TinkerGraph.open();
+ graph.io(GryoIo.build()).readGraph(GryoResourceAccess.class.getResource("grateful-dead.kryo").getFile());
+ vertices = IteratorUtils.list(graph.vertices());
+ } catch (final IOException e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ } else
+ throw new IllegalArgumentException("No legal toy graph was provided to load: " + configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH));
+
+ return sparkContext.parallelize(vertices.stream().map(VertexWritable::new).collect(Collectors.toList())).mapToPair(vertex -> new Tuple2<>(vertex.get().id(), vertex));
+ }
+}
[17/31] incubator-tinkerpop git commit: updated CHANGELOG again.
Posted by sp...@apache.org.
updated CHANGELOG again.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/447d7056
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/447d7056
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/447d7056
Branch: refs/heads/TINKERPOP3-860
Commit: 447d7056ef84865efe76d09033f4bd13b1ec0884
Parents: 461be03
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Thu Dec 3 07:51:22 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Dec 3 12:47:53 2015 -0700
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/447d7056/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 90c9f27..869b494 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,6 +26,7 @@ image::https://raw.githubusercontent.com/apache/incubator-tinkerpop/master/docs/
TinkerPop 3.1.1 (NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* Added `InputRDDFormat` which wraps an `InputRDD` to make it accessible to Hadoop and not just Spark.
* Added `AbstractSparkTest` which handles closing `SparkContext` instances between tests now that we support persisted contexts.
* Fixed a serialization bug in `GryoSerializer` that made it difficult for graph providers to yield `InputRDDs` for `SparkGraphComputer`.
* `SparkGraphComputer` is now tested against Gryo, GraphSON, and `InputRDD` data sources.
[13/31] incubator-tinkerpop git commit: Column enum now implements
Function. SelectColumnStep goes away as now select(column) is simply
(un(ddadhe covers) map(column) -- bam. Order.valueDecr/etc. goes away in
favor of order().by(values, decr). This means
Posted by sp...@apache.org.
Column enum now implements Function. SelectColumnStep goes away as now select(column) is simply (un(ddadhe covers) map(column) -- bam. Order.valueDecr/etc. goes away in favor of order().by(values,decr). This means we have decoupled the selector from the comparator so people can do other things besides decr and incr on values.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/b7c6041c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/b7c6041c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/b7c6041c
Branch: refs/heads/TINKERPOP3-860
Commit: b7c6041cf01fce0258f11faa1e405364e606fbf2
Parents: 0e43bdd
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Nov 24 15:04:02 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Dec 3 12:47:52 2015 -0700
----------------------------------------------------------------------
docs/src/reference/the-traversal.asciidoc | 16 ++---
.../gremlin/process/traversal/Order.java | 28 ++++++--
.../traversal/dsl/graph/GraphTraversal.java | 12 ++--
.../traversal/step/map/SelectColumnStep.java | 68 --------------------
.../traversal/step/util/FunctionComparator.java | 56 ++++++++++++++++
.../tinkerpop/gremlin/structure/Column.java | 37 ++++++++++-
.../process/traversal/step/map/OrderTest.java | 3 +-
.../structure/TinkerGraphPlayTest.java | 7 +-
8 files changed, 137 insertions(+), 90 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b7c6041c/docs/src/reference/the-traversal.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/the-traversal.asciidoc b/docs/src/reference/the-traversal.asciidoc
index 8d0b2fb..6ce4541 100644
--- a/docs/src/reference/the-traversal.asciidoc
+++ b/docs/src/reference/the-traversal.asciidoc
@@ -1573,11 +1573,11 @@ ranking.
graph.io(graphml()).readGraph('data/grateful-dead.xml')
g = graph.traversal()
g.V().hasLabel('song').out('followedBy').groupCount().by('name').
- order(local).by(valueDecr).limit(local, 5)
+ order(local).by(values,decr).limit(local, 5)
g.V().hasLabel('song').out('followedBy').groupCount().by('name').
- order(local).by(valueDecr).limit(local, 5).select(keys)
+ order(local).by(values,decr).limit(local, 5).select(keys)
g.V().hasLabel('song').out('followedBy').groupCount().by('name').
- order(local).by(valueDecr).limit(local, 5).select(keys).unfold()
+ order(local).by(values,decr).limit(local, 5).select(keys).unfold()
----
Similarly, for extracting the values from a path or map.
@@ -1589,7 +1589,7 @@ g = graph.traversal()
g.V().hasLabel('song').out('sungBy').groupCount().by('name') <1>
g.V().hasLabel('song').out('sungBy').groupCount().by('name').select(values) <2>
g.V().hasLabel('song').out('sungBy').groupCount().by('name').select(values).unfold().
- groupCount().order(local).by(valueDecr).limit(local, 5) <3>
+ groupCount().order(local).by(values,decr).limit(local, 5) <3>
----
<1> Which artist sung how many songs?
@@ -1779,10 +1779,10 @@ that can be used to time execution of a body of code.
[gremlin-groovy,modern]
----
-g.V().repeat(both().groupCount('m')).times(16).cap('m').order(local).by(valueDecr).next()
-clock(1) {g.V().repeat(both().groupCount('m')).times(16).cap('m').order(local).by(valueDecr).next()}
-g.V().repeat(timeLimit(2).both().groupCount('m')).times(16).cap('m').order(local).by(valueDecr).next()
-clock(1) {g.V().repeat(timeLimit(2).both().groupCount('m')).times(16).cap('m').order(local).by(valueDecr).next()}
+g.V().repeat(both().groupCount('m')).times(16).cap('m').order(local).by(values,decr).next()
+clock(1) {g.V().repeat(both().groupCount('m')).times(16).cap('m').order(local).by(values,decr).next()}
+g.V().repeat(timeLimit(2).both().groupCount('m')).times(16).cap('m').order(local).by(values,decr).next()
+clock(1) {g.V().repeat(timeLimit(2).both().groupCount('m')).times(16).cap('m').order(local).by(values,decr).next()}
----
In essence, the relative order is respected, even through the number of traversers at each vertex is not. The primary
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b7c6041c/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/Order.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/Order.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/Order.java
index e04863b..4e3801a 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/Order.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/Order.java
@@ -47,7 +47,12 @@ public enum Order implements Comparator<Object> {
public Order reversed() {
return incr;
}
- }, keyIncr {
+ },
+ /**
+ * @deprecated Use {@link org.apache.tinkerpop.gremlin.structure.Column#keys};
+ */
+ @Deprecated
+ keyIncr {
@Override
public int compare(final Object first, final Object second) {
return Comparator.<Comparable>naturalOrder().compare(((Map.Entry<Comparable, ?>) first).getKey(), ((Map.Entry<Comparable, ?>) second).getKey());
@@ -57,7 +62,12 @@ public enum Order implements Comparator<Object> {
public Order reversed() {
return keyDecr;
}
- }, valueIncr {
+ },
+ /**
+ * @deprecated Use {@link org.apache.tinkerpop.gremlin.structure.Column#values};
+ */
+ @Deprecated
+ valueIncr {
@Override
public int compare(final Object first, final Object second) {
return Comparator.<Comparable>naturalOrder().compare(((Map.Entry<?, Comparable>) first).getValue(), ((Map.Entry<?, Comparable>) second).getValue());
@@ -67,7 +77,12 @@ public enum Order implements Comparator<Object> {
public Order reversed() {
return valueDecr;
}
- }, keyDecr {
+ },
+ /**
+ * @deprecated Use {@link org.apache.tinkerpop.gremlin.structure.Column#keys};
+ */
+ @Deprecated
+ keyDecr {
@Override
public int compare(final Object first, final Object second) {
return Comparator.<Comparable>reverseOrder().compare(((Map.Entry<Comparable, ?>) first).getKey(), ((Map.Entry<Comparable, ?>) second).getKey());
@@ -77,7 +92,12 @@ public enum Order implements Comparator<Object> {
public Order reversed() {
return keyIncr;
}
- }, valueDecr {
+ },
+ /**
+ * @deprecated Use {@link org.apache.tinkerpop.gremlin.structure.Column#values};
+ */
+ @Deprecated
+ valueDecr {
@Override
public int compare(final Object first, final Object second) {
return Comparator.<Comparable>reverseOrder().compare(((Map.Entry<?, Comparable>) first).getValue(), ((Map.Entry<?, Comparable>) second).getValue());
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b7c6041c/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java
index 97d0c53..3673bce 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java
@@ -99,7 +99,6 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertyValueStep
import org.apache.tinkerpop.gremlin.process.traversal.step.map.RangeLocalStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.SackStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.SampleLocalStep;
-import org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectColumnStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectOneStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.SumGlobalStep;
@@ -128,6 +127,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.TraversalS
import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.TreeSideEffectStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.ElementFunctionComparator;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.ElementValueComparator;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.FunctionComparator;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.HasContainer;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.TraversalComparator;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.Tree;
@@ -472,11 +472,11 @@ public interface GraphTraversal<S, E> extends Traversal<S, E> {
}
public default <E2> GraphTraversal<S, Collection<E2>> select(final Column column) {
- return this.asAdmin().addStep(new SelectColumnStep<>(this.asAdmin(), column));
+ return this.map(new FunctionTraverser<>((Function) column));
}
/**
- * @deprecated As of release 3.1.0, replaced by {@link #select(SelectColumnStep.Column)}
+ * @deprecated As of release 3.1.0, replaced by {@link GraphTraversal#select(Column)}
*/
@Deprecated
public default <E2> GraphTraversal<S, E2> mapValues() {
@@ -484,7 +484,7 @@ public interface GraphTraversal<S, E> extends Traversal<S, E> {
}
/**
- * @deprecated As of release 3.1.0, replaced by {@link #select(SelectColumnStep.Column)}
+ * @deprecated As of release 3.1.0, replaced by {@link GraphTraversal#select(Column)}
*/
@Deprecated
public default <E2> GraphTraversal<S, E2> mapKeys() {
@@ -1198,6 +1198,10 @@ public interface GraphTraversal<S, E> extends Traversal<S, E> {
return this.by((Comparator) order);
}
+ public default <V> GraphTraversal<S, E> by(final Column column, final Comparator<V> objectComparator) {
+ return this.by(new FunctionComparator(column, objectComparator));
+ }
+
public default <V> GraphTraversal<S, E> by(final Function<Element, V> elementFunctionProjection,
final Comparator<V> elementFunctionValueComparator) {
return this.by((Comparator) new ElementFunctionComparator<>(elementFunctionProjection, elementFunctionValueComparator));
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b7c6041c/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/SelectColumnStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/SelectColumnStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/SelectColumnStep.java
deleted file mode 100644
index 1c28003..0000000
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/SelectColumnStep.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.process.traversal.step.map;
-
-import org.apache.tinkerpop.gremlin.structure.Column;
-import org.apache.tinkerpop.gremlin.process.traversal.Path;
-import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
-import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
-import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
-import org.apache.tinkerpop.gremlin.util.function.ArrayListSupplier;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class SelectColumnStep<S, E> extends MapStep<S, Collection<E>> {
-
- private final Column column;
-
- public SelectColumnStep(final Traversal.Admin traversal, final Column column) {
- super(traversal);
- this.column = column;
- }
-
- @Override
- protected Collection<E> map(Traverser.Admin<S> traverser) {
- final S start = traverser.get();
- if (start instanceof Map)
- return this.column.equals(Column.keys) ? ((Map<E, ?>) start).keySet() : ((Map<?, E>) start).values();
- else if (start instanceof Path)
- return (Collection<E>) (this.column.equals(Column.keys) ? new ArrayList<>(((Path) start).labels()) : new ArrayList(((Path) start).objects())); // necessary for serialization in complex GraphComputers (find fix)
- else if (start instanceof Map.Entry) // TODO: remove support for this?
- return Collections.singleton(this.column.equals(Column.keys) ? ((Map.Entry<E, ?>) start).getKey() : ((Map.Entry<?, E>) start).getValue());
- else
- throw new IllegalStateException("The traverser does not reference a map or path: " + traverser);
- }
-
- @Override
- public String toString() {
- return StringFactory.stepString(this, this.column);
- }
-
- @Override
- public int hashCode() {
- return super.hashCode() ^ this.column.hashCode();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b7c6041c/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/FunctionComparator.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/FunctionComparator.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/FunctionComparator.java
new file mode 100644
index 0000000..a133e38
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/FunctionComparator.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.process.traversal.step.util;
+
+import java.io.Serializable;
+import java.util.Comparator;
+import java.util.function.Function;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class FunctionComparator<A, B> implements Comparator<A>, Serializable {
+
+ private final Function<A, B> function;
+ private final Comparator<B> comparator;
+
+ public FunctionComparator(final Function<A, B> function, final Comparator<B> comparator) {
+ this.function = function;
+ this.comparator = comparator;
+ }
+
+ public Function<A, B> getFunction() {
+ return this.function;
+ }
+
+ public Comparator<B> getComparator() {
+ return this.comparator;
+ }
+
+ @Override
+ public int compare(final A first, final A second) {
+ return this.comparator.compare(this.function.apply(first), this.function.apply(second));
+ }
+
+ @Override
+ public String toString() {
+ return this.comparator.toString() + "(" + this.function + ')';
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b7c6041c/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Column.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Column.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Column.java
index 1ab49fa..fca90d7 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Column.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Column.java
@@ -19,19 +19,50 @@
package org.apache.tinkerpop.gremlin.structure;
+import org.apache.tinkerpop.gremlin.process.traversal.Path;
+
+import java.util.Map;
+import java.util.function.Function;
+
/**
* Column references a particular type of column in a complex data structure such as a {@link java.util.Map} or a {@link org.apache.tinkerpop.gremlin.process.traversal.Path}.
*
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-public enum Column {
+public enum Column implements Function<Object, Object> {
/**
* The keys associated with the data structure.
*/
- keys,
+ keys {
+ @Override
+ public Object apply(final Object object) {
+ if (object instanceof Map)
+ return ((Map) object).keySet();
+ else if (object instanceof Map.Entry)
+ return ((Map.Entry) object).getKey();
+ else if (object instanceof Path)
+ return ((Path) object).labels();
+ else
+ throw new IllegalArgumentException("The provided object does not have accessible keys: " + object.getClass());
+ }
+ },
/**
* The values associated with the data structure.
*/
- values
+ values {
+ @Override
+ public Object apply(final Object object) {
+ if (object instanceof Map)
+ return ((Map) object).values();
+ else if (object instanceof Map.Entry)
+ return ((Map.Entry) object).getValue();
+ else if (object instanceof Path)
+ return ((Path) object).objects();
+ else
+ throw new IllegalArgumentException("The provided object does not have accessible keys: " + object.getClass());
+ }
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b7c6041c/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderTest.java
index 2eec7ac..3f08d60 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderTest.java
@@ -27,6 +27,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.Scope;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalEngine;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.apache.tinkerpop.gremlin.structure.Column;
import org.apache.tinkerpop.gremlin.structure.T;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.junit.Test;
@@ -363,7 +364,7 @@ public abstract class OrderTest extends AbstractGremlinProcessTest {
map.put(3, (int) v.get().value("age") * 3);
map.put(4, (int) v.get().value("age"));
return map;
- }).order(Scope.local).by(Order.valueDecr).by(Order.keyIncr);
+ }).order(Scope.local).by(Column.values, Order.decr).by(Column.keys,Order.incr);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b7c6041c/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphPlayTest.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphPlayTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphPlayTest.java
index fdb44ad..64c2e84 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphPlayTest.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphPlayTest.java
@@ -20,13 +20,14 @@ package org.apache.tinkerpop.gremlin.tinkergraph.structure;
import org.apache.tinkerpop.gremlin.process.computer.bulkloading.BulkLoaderVertexProgram;
import org.apache.tinkerpop.gremlin.process.traversal.Operator;
+import org.apache.tinkerpop.gremlin.process.traversal.Order;
import org.apache.tinkerpop.gremlin.process.traversal.P;
import org.apache.tinkerpop.gremlin.process.traversal.Scope;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
-import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalMetrics;
+import org.apache.tinkerpop.gremlin.structure.Column;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.T;
import org.apache.tinkerpop.gremlin.structure.Vertex;
@@ -39,6 +40,7 @@ import org.junit.Test;
import java.io.File;
import java.util.Arrays;
import java.util.List;
+import java.util.function.Function;
import java.util.function.Supplier;
import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.*;
@@ -53,7 +55,8 @@ public class TinkerGraphPlayTest {
public void testPlay8() throws Exception {
Graph graph = TinkerFactory.createModern();
GraphTraversalSource g = graph.traversal(); //GraphTraversalSource.computer());
- System.out.println(g.V().outE("knows").identity().inV().count().is(P.eq(5)).explain());
+ //System.out.println(g.V().outE("knows").identity().inV().count().is(P.eq(5)).explain());
+ System.out.println(g.V().both().both().groupCount().order(Scope.local).by(Column.values, Order.incr).toList());
}
[24/31] incubator-tinkerpop git commit: Merge branch 'TINKERPOP-992'
Posted by sp...@apache.org.
Merge branch 'TINKERPOP-992'
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/00a2965f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/00a2965f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/00a2965f
Branch: refs/heads/TINKERPOP3-860
Commit: 00a2965f7380d274c2ba00a4b752865be07a9566
Parents: d59e914 937ebbc
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Thu Dec 3 15:29:02 2015 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Dec 3 15:29:02 2015 -0500
----------------------------------------------------------------------
.../org/apache/tinkerpop/gremlin/structure/io/IoGraphTest.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[31/31] incubator-tinkerpop git commit: Merge remote-tracking branch
'origin/master' into TINKERPOP3-860
Posted by sp...@apache.org.
Merge remote-tracking branch 'origin/master' into TINKERPOP3-860
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/b27a30b2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/b27a30b2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/b27a30b2
Branch: refs/heads/TINKERPOP3-860
Commit: b27a30b2eb20990e6d81932648d4f3ddec948ddf
Parents: 96cb4d6 3ac67f2
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Dec 4 09:20:20 2015 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Fri Dec 4 09:20:20 2015 -0500
----------------------------------------------------------------------
.travis.yml | 3 +-
CHANGELOG.asciidoc | 8 +
docs/src/dev/developer/contributing.asciidoc | 13 +-
docs/src/reference/the-traversal.asciidoc | 33 ++-
giraph-gremlin/pom.xml | 18 +-
.../src/test/resources/log4j-silent.properties | 18 ++
.../src/test/resources/log4j-test.properties | 23 ++
gremlin-console/pom.xml | 6 +
.../src/test/resources/log4j-silent.properties | 18 ++
.../src/test/resources/log4j-test.properties | 21 ++
gremlin-core/pom.xml | 6 +
.../gremlin/process/traversal/Order.java | 28 ++-
.../traversal/dsl/graph/GraphTraversal.java | 20 +-
.../traversal/step/map/SelectColumnStep.java | 68 ------
.../traversal/step/util/FunctionComparator.java | 56 +++++
.../tinkerpop/gremlin/structure/Column.java | 42 +++-
.../src/test/resources/log4j-silent.properties | 18 ++
.../src/test/resources/log4j-test.properties | 21 ++
.../src/test/resources/log4j-silent.properties | 18 ++
.../src/test/resources/log4j-test.properties | 21 ++
.../step/branch/GroovyLocalTest.groovy | 2 +-
.../traversal/step/map/GroovyOrderTest.groovy | 8 +-
.../process/GroovyProcessComputerSuite.java | 2 +
.../process/GroovyProcessStandardSuite.java | 2 +
.../src/test/resources/log4j-silent.properties | 18 ++
.../src/test/resources/log4j-test.properties | 21 ++
.../GremlinServerAuthIntegrateOldTest.java | 235 -------------------
.../GremlinServerAuthOldIntegrateTest.java | 235 +++++++++++++++++++
.../src/test/resources/log4j-silent.properties | 18 ++
.../src/test/resources/log4j-test.properties | 21 ++
gremlin-test/pom.xml | 6 +
.../tinkerpop/gremlin/AbstractGremlinTest.java | 6 +
.../BulkLoaderVertexProgramTest.java | 2 +-
.../process/traversal/step/map/OrderTest.java | 11 +-
.../gremlin/structure/io/IoGraphTest.java | 4 +-
.../tinkerpop/gremlin/structure/io/IoTest.java | 87 ++++---
.../src/test/resources/log4j-silent.properties | 21 ++
.../src/test/resources/log4j-test.properties | 24 ++
hadoop-gremlin/pom.xml | 18 +-
.../structure/hdfs/HadoopEdgeIterator.java | 4 -
.../structure/hdfs/HadoopElementIterator.java | 35 +--
.../structure/hdfs/HadoopVertexIterator.java | 4 -
.../groovy/plugin/HadoopGremlinPluginCheck.java | 176 ++++++++++++++
.../groovy/plugin/HadoopGremlinPluginTest.java | 173 --------------
.../hadoop/groovy/plugin/HadoopPluginSuite.java | 2 +-
.../HadoopGraphStructureStandardTest.java | 35 +++
.../io/HadoopGraphStructureStandardTest.java | 35 ---
.../src/test/resources/log4j-silent.properties | 18 ++
.../src/test/resources/log4j-test.properties | 21 ++
neo4j-gremlin/pom.xml | 6 +
.../src/test/resources/log4j-silent.properties | 18 ++
.../src/test/resources/log4j-test.properties | 21 ++
pom.xml | 55 ++---
spark-gremlin/pom.xml | 18 +-
.../process/computer/SparkContextHelper.java | 39 +++
.../process/computer/SparkGraphComputer.java | 15 +-
.../computer/payload/MessagePayload.java | 5 +-
.../computer/payload/ViewIncomingPayload.java | 6 +-
.../computer/payload/ViewOutgoingPayload.java | 12 +-
.../spark/structure/io/InputRDDFormat.java | 136 +++++++++++
.../spark/structure/io/gryo/GryoSerializer.java | 6 +
.../io/gryo/WrappedArraySerializer.java | 46 ++++
.../gremlin/spark/AbstractSparkTest.java | 43 ++++
.../process/computer/LocalPropertyTest.java | 8 +-
.../computer/SparkHadoopGraphProvider.java | 20 +-
.../spark/structure/io/InputOutputRDDTest.java | 7 +-
.../spark/structure/io/InputRDDTest.java | 28 ++-
.../spark/structure/io/OutputRDDTest.java | 7 +-
.../io/PersistedInputOutputRDDTest.java | 14 +-
.../spark/structure/io/ToyGraphInputRDD.java | 73 ++++++
.../src/test/resources/log4j-silent.properties | 18 ++
.../src/test/resources/log4j-test.properties | 25 ++
.../structure/TinkerGraphPlayTest.java | 7 +-
.../src/test/resources/log4j-silent.properties | 18 ++
.../src/test/resources/log4j-test.properties | 21 ++
75 files changed, 1655 insertions(+), 720 deletions(-)
----------------------------------------------------------------------
[15/31] incubator-tinkerpop git commit: GroovyProcessStartard was
missing GroovyFlatMapTest. Added.
Posted by sp...@apache.org.
GroovyProcessStartard was missing GroovyFlatMapTest. Added.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/5fda89ab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/5fda89ab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/5fda89ab
Branch: refs/heads/TINKERPOP3-860
Commit: 5fda89ab9cb58a54be5c2622a10d6fad9b942271
Parents: f820935
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed Dec 2 15:49:42 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Dec 3 12:47:53 2015 -0700
----------------------------------------------------------------------
.../tinkerpop/gremlin/process/GroovyProcessStandardSuite.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/5fda89ab/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessStandardSuite.java
----------------------------------------------------------------------
diff --git a/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessStandardSuite.java b/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessStandardSuite.java
index ae91b5f..7024869 100644
--- a/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessStandardSuite.java
+++ b/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessStandardSuite.java
@@ -48,6 +48,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyAddVertexTe
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyCoalesceTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyConstantTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyCountTest;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyFlatMapTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyFoldTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyGraphTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyLoopsTest;
@@ -128,6 +129,7 @@ public class GroovyProcessStandardSuite extends ProcessStandardSuite {
GroovyConstantTest.Traversals.class,
GroovyCountTest.Traversals.class,
GroovyFoldTest.Traversals.class,
+ GroovyFlatMapTest.Traversals.class,
GroovyGraphTest.Traversals.class,
GroovyLoopsTest.Traversals.class,
GroovyMapTest.Traversals.class,
[05/31] incubator-tinkerpop git commit: Remove the "3" from
"TINKERPOP3" in direct JIRA links.
Posted by sp...@apache.org.
Remove the "3" from "TINKERPOP3" in direct JIRA links.
Left direct links to issues alone. They will redirect.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/cdc063a8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/cdc063a8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/cdc063a8
Branch: refs/heads/TINKERPOP3-860
Commit: cdc063a8e3faa80fa15c1224298eded9b400b7d6
Parents: 978bf4e
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Thu Dec 3 05:51:45 2015 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Dec 3 05:51:45 2015 -0500
----------------------------------------------------------------------
docs/src/dev/developer/contributing.asciidoc | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/cdc063a8/docs/src/dev/developer/contributing.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/dev/developer/contributing.asciidoc b/docs/src/dev/developer/contributing.asciidoc
index d4b40dd..70c78e2 100644
--- a/docs/src/dev/developer/contributing.asciidoc
+++ b/docs/src/dev/developer/contributing.asciidoc
@@ -26,7 +26,7 @@ Getting Started
New contributors can start development with TinkerPop by first link:https://help.github.com/articles/fork-a-repo/[forking
then cloning] the Apache TinkerPop link:https://github.com/apache/incubator-tinkerpop[GitHub repository]. Generally
-speaking it is best to tie any work done to an issue in link:https://issues.apache.org/jira/browse/TINKERPOP3[JIRA].
+speaking it is best to tie any work done to an issue in link:https://issues.apache.org/jira/browse/TINKERPOP[JIRA].
Either scan through JIRA for an existing open issue to work on or create a new one.
NOTE: For those who are trying to find a place to start to contribute, consider looking at unresolved issues that
@@ -177,7 +177,7 @@ tags, as they produce permanent clutter.
Issue Tracker Conventions
~~~~~~~~~~~~~~~~~~~~~~~~~
-TinkerPop uses Apache JIRA as its link:https://issues.apache.org/jira/browse/TINKERPOP3[issue tracker]. JIRA is a
+TinkerPop uses Apache JIRA as its link:https://issues.apache.org/jira/browse/TINKERPOP[issue tracker]. JIRA is a
very robust piece of software with many options and configurations. To simplify usage and ensure consistency across
issues, the following conventions should be adhered to:
@@ -222,7 +222,7 @@ public APIs that might be utilized by users or implemented by vendors:
** `@deprecated As of release x.y.z, replaced by {@link SomeOtherClass#someNewMethod()}` - if the method is not
replaced then the comment can simply read "not replaced". Additional comments that provide more context are
encouraged.
-** `@see <a href="https://issues.apache.org/jira/browse/TINKERPOP3-XXX">TINKERPOP3-XXX</a>` - supply a link to the
+** `@see <a href="https://issues.apache.org/jira/browse/TINKERPOP-XXX">TINKERPOP-XXX</a>` - supply a link to the
JIRA issue for reference.
* All deprecation should typically be tied to a JIRA issue with a "breaking" label - the issue itself does not need to
specifically or solely be about "deprecation" but it should be documented very clearly in the comments what was
[18/31] incubator-tinkerpop git commit: updated CHANGELOG.
Posted by sp...@apache.org.
updated CHANGELOG.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/461be033
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/461be033
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/461be033
Branch: refs/heads/TINKERPOP3-860
Commit: 461be033b6b11ef97c0ae7af0b88d1beb1a326f9
Parents: 87382a9
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Thu Dec 3 07:49:40 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Dec 3 12:47:53 2015 -0700
----------------------------------------------------------------------
CHANGELOG.asciidoc | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/461be033/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 9f45a90..90c9f27 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,6 +26,10 @@ image::https://raw.githubusercontent.com/apache/incubator-tinkerpop/master/docs/
TinkerPop 3.1.1 (NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* Added `AbstractSparkTest` which handles closing `SparkContext` instances between tests now that we support persisted contexts.
+* Fixed a serialization bug in `GryoSerializer` that made it difficult for graph providers to yield `InputRDDs` for `SparkGraphComputer`.
+* `SparkGraphComputer` is now tested against Gryo, GraphSON, and `InputRDD` data sources.
+* `HadoopElementIterator` (for Hadoop-Gremlin OLTP) now works for any `InputFormat`, not just `FileInputFormats`.
* Added `Traverser.Admin.getTags()` which are used to mark branches in a traversal (useful in `match()` and related future steps).
* Fixed the `Future` model for `GiraphGraphComputer` and `SparkGraphComputer` so that class loaders are preserved.
* Added support for arbitrary vertex ID types in `BulkLoaderVertexProgram`.
[06/31] incubator-tinkerpop git commit: Provide the right expected
schema on graph configuration in IoGraphTest.
Posted by sp...@apache.org.
Provide the right expected schema on graph configuration in IoGraphTest.
Without this change, the graph woudl either create the wrong schema or not create one at all. Either would be a problem for graphs with strong typing.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/937ebbc4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/937ebbc4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/937ebbc4
Branch: refs/heads/TINKERPOP3-860
Commit: 937ebbc495803840cf26aec9b410fc3577a1251c
Parents: cdc063a
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Thu Dec 3 06:29:43 2015 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Dec 3 06:29:43 2015 -0500
----------------------------------------------------------------------
.../org/apache/tinkerpop/gremlin/structure/io/IoGraphTest.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/937ebbc4/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/IoGraphTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/IoGraphTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/IoGraphTest.java
index bc1296a..fece4a9 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/IoGraphTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/IoGraphTest.java
@@ -100,7 +100,7 @@ public class IoGraphTest extends AbstractGremlinTest {
try {
graph.io(ioBuilderToTest).writeGraph(f.getAbsolutePath());
- final Configuration configuration = graphProvider.newGraphConfiguration("readGraph", this.getClass(), name.getMethodName(), LoadGraphWith.GraphData.MODERN);
+ final Configuration configuration = graphProvider.newGraphConfiguration("readGraph", this.getClass(), name.getMethodName(), LoadGraphWith.GraphData.CLASSIC);
final Graph g1 = graphProvider.openTestGraph(configuration);
g1.io(ioBuilderToTest).readGraph(f.getAbsolutePath());
@@ -118,7 +118,7 @@ public class IoGraphTest extends AbstractGremlinTest {
@FeatureRequirement(featureClass = Graph.Features.EdgeFeatures.class, feature = Graph.Features.EdgeFeatures.FEATURE_ADD_EDGES)
@FeatureRequirement(featureClass = Graph.Features.VertexFeatures.class, feature = Graph.Features.VertexFeatures.FEATURE_ADD_VERTICES)
public void shouldMigrateClassicGraph() throws Exception {
- final Configuration configuration = graphProvider.newGraphConfiguration("readGraph", this.getClass(), name.getMethodName(), null);
+ final Configuration configuration = graphProvider.newGraphConfiguration("readGraph", this.getClass(), name.getMethodName(), LoadGraphWith.GraphData.CLASSIC);
graphProvider.clear(configuration);
final Graph g1 = graphProvider.openTestGraph(configuration);
[23/31] incubator-tinkerpop git commit: Loosened up assertions on
types for toy graphs.
Posted by sp...@apache.org.
Loosened up assertions on types for toy graphs.
This allowed us to not have to add a supportsSchema() feature which seemed kinda excessive to add for just a single tests. There are many other tests in IO that validat types strictly so this shouldn't be a problem to loosen stuff up here a bit.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/d59e9144
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/d59e9144
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/d59e9144
Branch: refs/heads/TINKERPOP3-860
Commit: d59e9144aa84fb274da32fbfb8d9742a3c89c43b
Parents: 0e43bdd
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Thu Dec 3 15:26:53 2015 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Dec 3 15:26:53 2015 -0500
----------------------------------------------------------------------
.../tinkerpop/gremlin/structure/io/IoTest.java | 87 +++++++++++++-------
1 file changed, 57 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d59e9144/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/IoTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/IoTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/IoTest.java
index 8f35333..40950f9 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/IoTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/IoTest.java
@@ -50,6 +50,8 @@ import org.junit.Before;
import org.junit.Test;
import org.junit.experimental.runners.Enclosed;
import org.junit.runner.RunWith;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import javax.xml.XMLConstants;
import javax.xml.transform.Source;
@@ -90,6 +92,7 @@ import static org.junit.Assert.fail;
*/
@RunWith(Enclosed.class)
public class IoTest {
+ private static final Logger logger = LoggerFactory.getLogger(IoTest.class);
public static class GraphMLTest extends AbstractGremlinTest {
@Test
@@ -648,7 +651,7 @@ public class IoTest {
assertEquals(6, IteratorUtils.count(g1.vertices()));
assertEquals(6, IteratorUtils.count(g1.edges()));
- final Vertex v1 = (Vertex) g1.traversal().V().has("name", "marko").next();
+ final Vertex v1 = g1.traversal().V().has("name", "marko").next();
assertEquals(29, v1.<Integer>value("age").intValue());
assertEquals(2, v1.keys().size());
assertEquals(assertSpecificLabel ? "person" : Vertex.DEFAULT_LABEL, v1.label());
@@ -660,25 +663,25 @@ public class IoTest {
if (e.inVertex().value("name").equals("vadas")) {
assertEquals("knows", e.label());
if (assertDouble)
- assertEquals(0.5d, e.value("weight"), 0.0001d);
+ assertWeightLoosely(0.5d, e);
else
- assertEquals(0.5f, e.value("weight"), 0.0001f);
+ assertWeightLoosely(0.5f, e);
assertEquals(1, e.keys().size());
assertId(g1, lossyForId, e, 7);
} else if (e.inVertex().value("name").equals("josh")) {
assertEquals("knows", e.label());
if (assertDouble)
- assertEquals(1.0, e.value("weight"), 0.0001d);
+ assertWeightLoosely(1.0, e);
else
- assertEquals(1.0f, e.value("weight"), 0.0001f);
+ assertWeightLoosely(1.0f, e);
assertEquals(1, e.keys().size());
assertId(g1, lossyForId, e, 8);
} else if (e.inVertex().value("name").equals("lop")) {
assertEquals("created", e.label());
if (assertDouble)
- assertEquals(0.4d, e.value("weight"), 0.0001d);
+ assertWeightLoosely(0.4d, e);
else
- assertEquals(0.4f, e.value("weight"), 0.0001f);
+ assertWeightLoosely(0.4f, e);
assertEquals(1, e.keys().size());
assertId(g1, lossyForId, e, 9);
} else {
@@ -686,7 +689,7 @@ public class IoTest {
}
});
- final Vertex v2 = (Vertex) g1.traversal().V().has("name", "vadas").next();
+ final Vertex v2 = g1.traversal().V().has("name", "vadas").next();
assertEquals(27, v2.<Integer>value("age").intValue());
assertEquals(2, v2.keys().size());
assertEquals(assertSpecificLabel ? "person" : Vertex.DEFAULT_LABEL, v2.label());
@@ -698,9 +701,9 @@ public class IoTest {
if (e.outVertex().value("name").equals("marko")) {
assertEquals("knows", e.label());
if (assertDouble)
- assertEquals(0.5d, e.value("weight"), 0.0001d);
+ assertWeightLoosely(0.5d, e);
else
- assertEquals(0.5f, e.value("weight"), 0.0001f);
+ assertWeightLoosely(0.5f, e);
assertEquals(1, e.keys().size());
assertId(g1, lossyForId, e, 7);
} else {
@@ -708,7 +711,7 @@ public class IoTest {
}
});
- final Vertex v3 = (Vertex) g1.traversal().V().has("name", "lop").next();
+ final Vertex v3 = g1.traversal().V().has("name", "lop").next();
assertEquals("java", v3.<String>value("lang"));
assertEquals(2, v2.keys().size());
assertEquals(assertSpecificLabel ? "software" : Vertex.DEFAULT_LABEL, v3.label());
@@ -720,25 +723,25 @@ public class IoTest {
if (e.outVertex().value("name").equals("peter")) {
assertEquals("created", e.label());
if (assertDouble)
- assertEquals(0.2d, e.value("weight"), 0.0001d);
+ assertWeightLoosely(0.2d, e);
else
- assertEquals(0.2f, e.value("weight"), 0.0001f);
+ assertWeightLoosely(0.2f, e);
assertEquals(1, e.keys().size());
assertId(g1, lossyForId, e, 12);
} else if (e.outVertex().value("name").equals("josh")) {
assertEquals("created", e.label());
if (assertDouble)
- assertEquals(0.4d, e.value("weight"), 0.0001d);
+ assertWeightLoosely(0.4d, e);
else
- assertEquals(0.4f, e.value("weight"), 0.0001f);
+ assertWeightLoosely(0.4f, e);
assertEquals(1, e.keys().size());
assertId(g1, lossyForId, e, 11);
} else if (e.outVertex().value("name").equals("marko")) {
assertEquals("created", e.label());
if (assertDouble)
- assertEquals(0.4d, e.value("weight"), 0.0001d);
+ assertWeightLoosely(0.4d, e);
else
- assertEquals(0.4f, e.value("weight"), 0.0001f);
+ assertWeightLoosely(0.4f, e);
assertEquals(1, e.keys().size());
assertId(g1, lossyForId, e, 9);
} else {
@@ -746,7 +749,7 @@ public class IoTest {
}
});
- final Vertex v4 = (Vertex) g1.traversal().V().has("name", "josh").next();
+ final Vertex v4 = g1.traversal().V().has("name", "josh").next();
assertEquals(32, v4.<Integer>value("age").intValue());
assertEquals(2, v4.keys().size());
assertEquals(assertSpecificLabel ? "person" : Vertex.DEFAULT_LABEL, v4.label());
@@ -758,25 +761,25 @@ public class IoTest {
if (e.inVertex().value("name").equals("ripple")) {
assertEquals("created", e.label());
if (assertDouble)
- assertEquals(1.0d, e.value("weight"), 0.0001d);
+ assertWeightLoosely(1.0d, e);
else
- assertEquals(1.0f, e.value("weight"), 0.0001f);
+ assertWeightLoosely(1.0f, e);
assertEquals(1, e.keys().size());
assertId(g1, lossyForId, e, 10);
} else if (e.inVertex().value("name").equals("lop")) {
assertEquals("created", e.label());
if (assertDouble)
- assertEquals(0.4d, e.value("weight"), 0.0001d);
+ assertWeightLoosely(0.4d, e);
else
- assertEquals(0.4f, e.value("weight"), 0.0001f);
+ assertWeightLoosely(0.4f, e);
assertEquals(1, e.keys().size());
assertId(g1, lossyForId, e, 11);
} else if (e.outVertex().value("name").equals("marko")) {
assertEquals("knows", e.label());
if (assertDouble)
- assertEquals(1.0d, e.value("weight"), 0.0001d);
+ assertWeightLoosely(1.0d, e);
else
- assertEquals(1.0f, e.value("weight"), 0.0001f);
+ assertWeightLoosely(1.0f, e);
assertEquals(1, e.keys().size());
assertId(g1, lossyForId, e, 8);
} else {
@@ -784,7 +787,7 @@ public class IoTest {
}
});
- final Vertex v5 = (Vertex) g1.traversal().V().has("name", "ripple").next();
+ final Vertex v5 = g1.traversal().V().has("name", "ripple").next();
assertEquals("java", v5.<String>value("lang"));
assertEquals(2, v5.keys().size());
assertEquals(assertSpecificLabel ? "software" : Vertex.DEFAULT_LABEL, v5.label());
@@ -796,9 +799,9 @@ public class IoTest {
if (e.outVertex().value("name").equals("josh")) {
assertEquals("created", e.label());
if (assertDouble)
- assertEquals(1.0d, e.value("weight"), 0.0001d);
+ assertWeightLoosely(1.0d, e);
else
- assertEquals(1.0f, e.value("weight"), 0.0001f);
+ assertWeightLoosely(1.0f, e);
assertEquals(1, e.keys().size());
assertId(g1, lossyForId, e, 10);
} else {
@@ -806,7 +809,7 @@ public class IoTest {
}
});
- final Vertex v6 = (Vertex) g1.traversal().V().has("name", "peter").next();
+ final Vertex v6 = g1.traversal().V().has("name", "peter").next();
assertEquals(35, v6.<Integer>value("age").intValue());
assertEquals(2, v6.keys().size());
assertEquals(assertSpecificLabel ? "person" : Vertex.DEFAULT_LABEL, v6.label());
@@ -818,9 +821,9 @@ public class IoTest {
if (e.inVertex().value("name").equals("lop")) {
assertEquals("created", e.label());
if (assertDouble)
- assertEquals(0.2d, e.value("weight"), 0.0001d);
+ assertWeightLoosely(0.2d, e);
else
- assertEquals(0.2f, e.value("weight"), 0.0001f);
+ assertWeightLoosely(0.2f, e);
assertEquals(1, e.keys().size());
assertId(g1, lossyForId, e, 12);
} else {
@@ -829,6 +832,30 @@ public class IoTest {
});
}
+ private static void assertWeightLoosely(final double expected, final Edge e) {
+ try {
+ assertEquals(expected, e.value("weight"), 0.0001d);
+ } catch (Exception ex) {
+ // for graphs that have strong typing via schema it is possible that a value that came across as graphson
+ // with lossiness will end up having a value expected to double to be coerced to float by the underlying
+ // graph.
+ logger.warn("Attempting to assert weight as float for {} - if your graph is strongly typed from schema this is likely expected", e);
+ assertEquals(new Double(expected).floatValue(), e.value("weight"), 0.0001f);
+ }
+ }
+
+ private static void assertWeightLoosely(final float expected, final Edge e) {
+ try {
+ assertEquals(expected, e.value("weight"), 0.0001f);
+ } catch (Exception ex) {
+ // for graphs that have strong typing via schema it is possible that a value that came across as graphson
+ // with lossiness will end up having a value expected to float to be coerced to double by the underlying
+ // graph.
+ logger.warn("Attempting to assert weight as double for {} - if your graph is strongly typed from schema this is likely expected", e);
+ assertEquals(new Float(expected).doubleValue(), e.value("weight"), 0.0001d);
+ }
+ }
+
private static void assertId(final Graph g, final boolean lossyForId, final Element e, final Object expected) {
// it is possible that a Graph (e.g. elastic-gremlin) can supportUserSuppliedIds but internally
// represent them as a value other than Numeric (which is what's in all of the test/toy data).
[02/31] incubator-tinkerpop git commit: Adjust tests to be a bit more
bulletproof with respect to thread timing.
Posted by sp...@apache.org.
Adjust tests to be a bit more bulletproof with respect to thread timing.
Not sure if i've fixed anything with this change as these tests have never failed for me on my system.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/87201658
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/87201658
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/87201658
Branch: refs/heads/TINKERPOP3-860
Commit: 87201658f2023c725d4620b4a700b949605099a7
Parents: 59eaa06
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Wed Dec 2 14:20:30 2015 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Wed Dec 2 14:20:49 2015 -0500
----------------------------------------------------------------------
.../groovy/engine/GremlinExecutorTest.java | 8 +--
.../jsr223/GremlinGroovyScriptEngineTest.java | 52 ++++++++++++--------
2 files changed, 35 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/87201658/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/GremlinExecutorTest.java
----------------------------------------------------------------------
diff --git a/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/GremlinExecutorTest.java b/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/GremlinExecutorTest.java
index c30ad79..7bc0708 100644
--- a/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/GremlinExecutorTest.java
+++ b/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/GremlinExecutorTest.java
@@ -478,7 +478,7 @@ public class GremlinExecutorTest {
public void shouldContinueToEvalScriptsEvenWithTimedInterrupt() throws Exception {
final Map<String,List<Object>> compilerCustomizerConfig = new HashMap<>();
final List<Object> args = new ArrayList<>();
- args.add(50);
+ args.add(250);
compilerCustomizerConfig.put(TimedInterruptCustomizerProvider.class.getName(), args);
final Map<String, Object> config = new HashMap<>();
@@ -494,14 +494,14 @@ public class GremlinExecutorTest {
for (int ix = 0; ix < 5; ix++) {
try {
- // this script takes 10 ms longer than the interruptionTimeout
- gremlinExecutor.eval("s = System.currentTimeMillis();\nwhile((System.currentTimeMillis() - s) < 100) {}").get();
+ // this script takes significantly longer than the interruptionTimeout
+ gremlinExecutor.eval("s = System.currentTimeMillis();\nwhile((System.currentTimeMillis() - s) < 10000) {}").get();
fail("This should have timed out");
} catch (Exception se) {
assertEquals(TimeoutException.class, se.getCause().getClass());
}
- // this script takes 10 ms less than the interruptionTimeout
+ // this script takes significantly less than the interruptionTimeout
assertEquals("test", gremlinExecutor.eval("s = System.currentTimeMillis();\nwhile((System.currentTimeMillis() - s) < 20) {};'test'").get());
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/87201658/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngineTest.java
----------------------------------------------------------------------
diff --git a/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngineTest.java b/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngineTest.java
index 9e74686..f3b3835 100644
--- a/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngineTest.java
+++ b/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngineTest.java
@@ -29,12 +29,17 @@ import javax.script.ScriptContext;
import javax.script.ScriptEngine;
import javax.script.ScriptException;
import javax.script.SimpleBindings;
+import java.awt.*;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.greaterThan;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -203,51 +208,56 @@ public class GremlinGroovyScriptEngineTest {
@Test
public void shouldReloadClassLoaderWhileDoingEvalInSeparateThread() throws Exception {
final AtomicBoolean fail = new AtomicBoolean(false);
+ final AtomicInteger counter = new AtomicInteger(0);
final CountDownLatch latch = new CountDownLatch(1);
+ final AtomicReference<Color> color = new AtomicReference<>(Color.RED);
+
final GremlinGroovyScriptEngine scriptEngine = new GremlinGroovyScriptEngine();
- final Thread t = new Thread(() -> {
- try {
- final Object o = scriptEngine.eval("Color.BLACK");
- System.out.println("Should not print: " + o);
- fail.set(true);
- } catch (ScriptException se) {
- // should get here as Color.BLACK is not imported yet.
- System.out.println("Failed to execute Color.BLACK as expected.");
- }
+ try {
+ scriptEngine.eval("Color.BLACK");
+ fail("Should fail as class is not yet imported");
+ } catch (ScriptException se) {
+ // should get here as Color.BLACK is not imported yet.
+ System.out.println("Failed to execute Color.BLACK as expected.");
+ }
+
+ final Thread evalThread = new Thread(() -> {
try {
- int counter = 0;
+ // execute scripts until the other thread releases this latch (i.e. after import)
while (latch.getCount() == 1) {
scriptEngine.eval("1+1");
- counter++;
+ counter.incrementAndGet();
}
- System.out.println(counter + " executions.");
-
- scriptEngine.eval("Color.BLACK");
- System.out.println("Color.BLACK now evaluates");
+ color.set((Color) scriptEngine.eval("Color.BLACK"));
} catch (Exception se) {
- se.printStackTrace();
fail.set(true);
}
}, "test-reload-classloader-1");
- t.start();
+ evalThread.start();
- // let the first thead execute a bit.
+ // let the first thread execute a bit.
Thread.sleep(1000);
- new Thread(() -> {
+ final Thread importThread = new Thread(() -> {
System.out.println("Importing java.awt.Color...");
final Set<String> imports = new HashSet<String>() {{
add("import java.awt.Color");
}};
scriptEngine.addImports(imports);
latch.countDown();
- }, "test-reload-classloader-2").start();
+ }, "test-reload-classloader-2");
+
+ importThread.start();
- t.join();
+ // block until both threads are done
+ importThread.join();
+ evalThread.join();
+ assertEquals(Color.BLACK, color.get());
+ assertThat(counter.get(), greaterThan(0));
assertFalse(fail.get());
}
[27/31] incubator-tinkerpop git commit: I merged the Hadoop PR and
then the Column PR. Column PR had a conflict in integration testing in Spark
cause of unregistered classes. Simple fix. CTR.
Posted by sp...@apache.org.
I merged the Hadoop PR and then the Column PR. Column PR had a conflict in integration testing in Spark cause of unregistered classes. Simple fix. CTR.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/04f7e248
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/04f7e248
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/04f7e248
Branch: refs/heads/TINKERPOP3-860
Commit: 04f7e248a5f1723bec935b6d29bf69871abccaf3
Parents: 19beece
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Fri Dec 4 05:22:28 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Fri Dec 4 05:22:28 2015 -0700
----------------------------------------------------------------------
CHANGELOG.asciidoc | 2 +-
.../org/apache/tinkerpop/gremlin/structure/Column.java | 11 +++++++----
2 files changed, 8 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f7e248/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 717aa71..1b2e1d9 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,7 +26,7 @@ image::https://raw.githubusercontent.com/apache/incubator-tinkerpop/master/docs/
TinkerPop 3.1.1 (NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-* `Column` now implements `Function` and now `by(valueDecr)` can be replaced with `by(values,decr)` and thus, projection and order are now separated.
+* `Column` now implements `Function`. The modulator `by(valueDecr)` can be replaced by `by(values,decr)` and thus, projection and order are separated.
* Added `InputRDDFormat` which wraps an `InputRDD` to make it accessible to Hadoop and not just Spark.
* Added `AbstractSparkTest` which handles closing `SparkContext` instances between tests now that we support persisted contexts.
* Fixed a serialization bug in `GryoSerializer` that made it difficult for graph providers to yield `InputRDDs` for `SparkGraphComputer`.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f7e248/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Column.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Column.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Column.java
index 38921c1..a20d7e7 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Column.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Column.java
@@ -21,6 +21,9 @@ package org.apache.tinkerpop.gremlin.structure;
import org.apache.tinkerpop.gremlin.process.traversal.Path;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
import java.util.Map;
import java.util.function.Function;
@@ -38,11 +41,11 @@ public enum Column implements Function<Object, Object> {
@Override
public Object apply(final Object object) {
if (object instanceof Map)
- return ((Map) object).keySet();
+ return new LinkedHashSet<>(((Map<?,?>) object).keySet());
else if (object instanceof Map.Entry)
return ((Map.Entry) object).getKey();
else if (object instanceof Path)
- return ((Path) object).labels();
+ return new ArrayList<>(((Path) object).labels());
else
throw new IllegalArgumentException("The provided object does not have accessible keys: " + object.getClass());
}
@@ -54,11 +57,11 @@ public enum Column implements Function<Object, Object> {
@Override
public Object apply(final Object object) {
if (object instanceof Map)
- return ((Map) object).values();
+ return new ArrayList<>(((Map<?,?>) object).values());
else if (object instanceof Map.Entry)
return ((Map.Entry) object).getValue();
else if (object instanceof Path)
- return ((Path) object).objects();
+ return new ArrayList<>(((Path) object).objects());
else
throw new IllegalArgumentException("The provided object does not have accessible keys: " + object.getClass());
}
[30/31] incubator-tinkerpop git commit: Merge remote-tracking branch
'origin/master'
Posted by sp...@apache.org.
Merge remote-tracking branch 'origin/master'
Conflicts:
CHANGELOG.asciidoc
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/3ac67f2c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/3ac67f2c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/3ac67f2c
Branch: refs/heads/TINKERPOP3-860
Commit: 3ac67f2c8295b02b107be8aad2f9661cd8b09e89
Parents: 2b2bf96 04f7e24
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Dec 4 09:09:28 2015 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Fri Dec 4 09:09:28 2015 -0500
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
.../org/apache/tinkerpop/gremlin/structure/Column.java | 11 +++++++----
2 files changed, 8 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/3ac67f2c/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --cc CHANGELOG.asciidoc
index 1fa03d5,1b2e1d9..9b3fa71
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@@ -26,8 -26,7 +26,9 @@@ image::https://raw.githubusercontent.co
TinkerPop 3.1.1 (NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* Improved logging control during builds with Maven.
+* `Column` now implements `Function` and now `by(valueDecr)` can be replaced with `by(values,decr)` and thus, projection and order are now separated.
+ * `Column` now implements `Function`. The modulator `by(valueDecr)` can be replaced by `by(values,decr)` and thus, projection and order are separated.
* Added `InputRDDFormat` which wraps an `InputRDD` to make it accessible to Hadoop and not just Spark.
* Added `AbstractSparkTest` which handles closing `SparkContext` instances between tests now that we support persisted contexts.
* Fixed a serialization bug in `GryoSerializer` that made it difficult for graph providers to yield `InputRDDs` for `SparkGraphComputer`.
[28/31] incubator-tinkerpop git commit: Improved the ability to
control logging for maven builds.
Posted by sp...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/spark-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/spark-gremlin/pom.xml b/spark-gremlin/pom.xml
index 8721c5f..63ac42f 100644
--- a/spark-gremlin/pom.xml
+++ b/spark-gremlin/pom.xml
@@ -347,6 +347,10 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
</plugin>
<plugin>
@@ -387,20 +391,6 @@
</archive>
</configuration>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <!-- log4j.configuration>log4j-core.properties</log4j.configuration> -->
- <!--<argLine>-Xmx2048M</argLine>-->
- <excludes>
- <exclude>**/*IntegrateTest.java</exclude>
- <exclude>**/*PerformanceTest.java</exclude>
- <!-- this is technically a member of the integration test suite -->
- <exclude>**/HadoopGremlinPluginTest.java</exclude>
- </excludes>
- </configuration>
- </plugin>
</plugins>
</build>
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/spark-gremlin/src/test/resources/log4j-silent.properties
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/resources/log4j-silent.properties b/spark-gremlin/src/test/resources/log4j-silent.properties
new file mode 100644
index 0000000..9c6148d
--- /dev/null
+++ b/spark-gremlin/src/test/resources/log4j-silent.properties
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+log4j.rootLogger=OFF
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/spark-gremlin/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/resources/log4j-test.properties b/spark-gremlin/src/test/resources/log4j-test.properties
new file mode 100644
index 0000000..fe1cb1e
--- /dev/null
+++ b/spark-gremlin/src/test/resources/log4j-test.properties
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+log4j.rootLogger=WARN, stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n
+
+log4j.logger.org.apache.spark=ERROR
+log4j.logger.org.spark-project=ERROR
+log4j.logger.org.apache.hadoop=ERROR
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/tinkergraph-gremlin/src/test/resources/log4j-silent.properties
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/test/resources/log4j-silent.properties b/tinkergraph-gremlin/src/test/resources/log4j-silent.properties
new file mode 100644
index 0000000..9c6148d
--- /dev/null
+++ b/tinkergraph-gremlin/src/test/resources/log4j-silent.properties
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+log4j.rootLogger=OFF
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/tinkergraph-gremlin/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/test/resources/log4j-test.properties b/tinkergraph-gremlin/src/test/resources/log4j-test.properties
new file mode 100644
index 0000000..ef436fe
--- /dev/null
+++ b/tinkergraph-gremlin/src/test/resources/log4j-test.properties
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+log4j.rootLogger=WARN, stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n
\ No newline at end of file
[09/31] incubator-tinkerpop git commit: Merge remote-tracking branch
'origin/master' into TINKERPOP3-860
Posted by sp...@apache.org.
Merge remote-tracking branch 'origin/master' into TINKERPOP3-860
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/96cb4d69
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/96cb4d69
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/96cb4d69
Branch: refs/heads/TINKERPOP3-860
Commit: 96cb4d69d14c237df5d40ccb1d8469607fa2ac7d
Parents: b503e41 0e43bdd
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Thu Dec 3 14:28:45 2015 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Dec 3 14:28:45 2015 -0500
----------------------------------------------------------------------
.travis.yml | 4 +-
CHANGELOG.asciidoc | 1 +
docs/src/dev/developer/contributing.asciidoc | 6 +--
.../gremlin/process/traversal/Traverser.java | 9 ++++
.../process/traversal/step/map/MatchStep.java | 56 +++++++++++++-------
.../traversal/step/util/ComputerAwareStep.java | 2 +-
.../traversal/traverser/O_Traverser.java | 37 +++++++++++++
.../traverser/util/EmptyTraverser.java | 6 +++
.../traversal/step/map/MatchStepTest.java | 15 ++++--
.../groovy/engine/GremlinExecutorTest.java | 8 +--
.../jsr223/GremlinGroovyScriptEngineTest.java | 51 ++++++++++--------
11 files changed, 138 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/96cb4d69/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngineTest.java
----------------------------------------------------------------------
[26/31] incubator-tinkerpop git commit: realized a stupid bug in
ToyGraphInputRDD around reduceByKey(). Fixed. Thanks @dalaro. CTR.
Posted by sp...@apache.org.
realized a stupid bug in ToyGraphInputRDD around reduceByKey(). Fixed. Thanks @dalaro. CTR.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/19beece8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/19beece8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/19beece8
Branch: refs/heads/TINKERPOP3-860
Commit: 19beece8f774f4b71a0895dd40e29a32c68762fd
Parents: 641a8cb
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Thu Dec 3 16:20:42 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Dec 3 16:20:42 2015 -0700
----------------------------------------------------------------------
.../gremlin/spark/process/computer/SparkHadoopGraphProvider.java | 2 --
.../tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java | 2 +-
2 files changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/19beece8/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
index de5a60d..618904a 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
@@ -48,10 +48,8 @@ public final class SparkHadoopGraphProvider extends HadoopGraphProvider {
config.put(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true); // this makes the test suite go really fast
if (null != loadGraphWith &&
!test.equals(BulkLoaderVertexProgramTest.class) &&
- !test.equals(PageRankVertexProgramTest.class) &&
RANDOM.nextBoolean()) {
config.put(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, ToyGraphInputRDD.class.getCanonicalName());
- config.put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName());
}
/// spark configuration
config.put("spark.master", "local[4]");
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/19beece8/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
index ea3636f..ff253e3 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
@@ -68,6 +68,6 @@ public final class ToyGraphInputRDD implements InputRDD {
} else
throw new IllegalArgumentException("No legal toy graph was provided to load: " + configuration.getProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION));
- return sparkContext.parallelize(vertices).mapToPair(vertex -> new Tuple2<>(vertex.get().id(), vertex));
+ return sparkContext.parallelize(vertices).mapToPair(vertex -> new Tuple2<>(vertex.get().id(), vertex)).reduceByKey((a, b) -> a); // if this is not done, then the graph is partitioned and you can have duplicate vertices;
}
}
[08/31] incubator-tinkerpop git commit: Merge remote-tracking branch
'origin/TINKERPOP3-1006'
Posted by sp...@apache.org.
Merge remote-tracking branch 'origin/TINKERPOP3-1006'
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/0e43bdd3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/0e43bdd3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/0e43bdd3
Branch: refs/heads/TINKERPOP3-860
Commit: 0e43bdd3ef8c65219b568aadcffd26be2e969c38
Parents: b70f3c7 8720165
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Thu Dec 3 14:24:03 2015 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Dec 3 14:24:03 2015 -0500
----------------------------------------------------------------------
.../groovy/engine/GremlinExecutorTest.java | 8 +--
.../jsr223/GremlinGroovyScriptEngineTest.java | 52 ++++++++++++--------
2 files changed, 35 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
[04/31] incubator-tinkerpop git commit: Updated CHANGELOG.
Posted by sp...@apache.org.
Updated CHANGELOG.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/978bf4eb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/978bf4eb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/978bf4eb
Branch: refs/heads/TINKERPOP3-860
Commit: 978bf4eb60ac18258ac4c22be6f30457bdb31a5d
Parents: 6041de6
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed Dec 2 16:14:40 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed Dec 2 16:14:40 2015 -0700
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/978bf4eb/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 5fdd59a..9f45a90 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,6 +26,7 @@ image::https://raw.githubusercontent.com/apache/incubator-tinkerpop/master/docs/
TinkerPop 3.1.1 (NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* Added `Traverser.Admin.getTags()` which are used to mark branches in a traversal (useful in `match()` and related future steps).
* Fixed the `Future` model for `GiraphGraphComputer` and `SparkGraphComputer` so that class loaders are preserved.
* Added support for arbitrary vertex ID types in `BulkLoaderVertexProgram`.
* Deprecate `credentialsDbLocation` from `SimpleAuthenticator` in Gremlin Server.
[14/31] incubator-tinkerpop git commit: HadoopElementIterator now
supports ANY InputFormat, not just FileInputFormat. Sweet. Also,
if you are using an RDD in Spark (and thus,
not really doing Hadoop InputFormat stuffs),
we have InputRDDFormat which wraps
Posted by sp...@apache.org.
HadoopElementIterator now supports ANY InputFormat, not just FileInputFormat. Sweet. Also, if you are using an RDD in Spark (and thus, not really doing Hadoop InputFormat stuffs), we have InputRDDFormat which wraps an RDD in an InputFormat so HadoopElementIterator works as well. This solves the HadoopGraph OLTP problem for ALL InputFormats and it allows ComputerResultStep to Attach elements for more than just FileInputFormats. Good stuff.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/85eb63db
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/85eb63db
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/85eb63db
Branch: refs/heads/TINKERPOP3-860
Commit: 85eb63dbc4855f7f3e944b82af8c9aae03c8aed2
Parents: 5b9de83
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Dec 1 09:25:14 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Dec 3 12:47:52 2015 -0700
----------------------------------------------------------------------
.../structure/hdfs/HadoopEdgeIterator.java | 4 -
.../structure/hdfs/HadoopElementIterator.java | 35 +++--
.../structure/hdfs/HadoopVertexIterator.java | 4 -
.../HadoopGraphStructureStandardTest.java | 35 +++++
.../io/HadoopGraphStructureStandardTest.java | 35 -----
.../spark/structure/io/InputRDDFormat.java | 136 +++++++++++++++++++
.../spark/structure/io/InputRDDTest.java | 21 +++
7 files changed, 208 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/85eb63db/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopEdgeIterator.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopEdgeIterator.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopEdgeIterator.java
index acdc0a4..59a4d2c 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopEdgeIterator.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopEdgeIterator.java
@@ -39,10 +39,6 @@ public final class HadoopEdgeIterator extends HadoopElementIterator<Edge> {
private Iterator<Edge> edgeIterator = Collections.emptyIterator();
- public HadoopEdgeIterator(final HadoopGraph graph, final InputFormat<NullWritable, VertexWritable> inputFormat, final Path path) throws IOException, InterruptedException {
- super(graph, inputFormat, path);
- }
-
public HadoopEdgeIterator(final HadoopGraph graph) throws IOException {
super(graph);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/85eb63db/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopElementIterator.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopElementIterator.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopElementIterator.java
index a9f287b..f9ffea2 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopElementIterator.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopElementIterator.java
@@ -19,13 +19,14 @@
package org.apache.tinkerpop.gremlin.hadoop.structure.hdfs;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptID;
-import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.task.JobContextImpl;
import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
@@ -36,35 +37,31 @@ import org.apache.tinkerpop.gremlin.structure.Element;
import java.io.IOException;
import java.util.Iterator;
import java.util.LinkedList;
+import java.util.List;
import java.util.Queue;
+import java.util.UUID;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
public abstract class HadoopElementIterator<E extends Element> implements Iterator<E> {
- // TODO: Generalize so it works for more than just FileFormats.
-
protected final HadoopGraph graph;
protected final Queue<RecordReader<NullWritable, VertexWritable>> readers = new LinkedList<>();
- public HadoopElementIterator(final HadoopGraph graph, final InputFormat<NullWritable, VertexWritable> inputFormat, final Path path) throws IOException, InterruptedException {
- this.graph = graph;
- final Configuration configuration = ConfUtil.makeHadoopConfiguration(this.graph.configuration());
- for (final Path path2 : HDFSTools.getAllFilePaths(FileSystem.get(configuration), path, HiddenFileFilter.instance())) {
- this.readers.add(inputFormat.createRecordReader(new FileSplit(path2, 0, Long.MAX_VALUE, new String[]{}), new TaskAttemptContextImpl(configuration, new TaskAttemptID())));
- }
- }
-
public HadoopElementIterator(final HadoopGraph graph) throws IOException {
try {
this.graph = graph;
- if (this.graph.configuration().containsKey(Constants.GREMLIN_HADOOP_INPUT_LOCATION)) {
- final Configuration configuration = ConfUtil.makeHadoopConfiguration(this.graph.configuration());
- final InputFormat<NullWritable, VertexWritable> inputFormat = this.graph.configuration().getGraphInputFormat().getConstructor().newInstance();
- for (final Path path : HDFSTools.getAllFilePaths(FileSystem.get(configuration), new Path(graph.configuration().getInputLocation()), HiddenFileFilter.instance())) {
- this.readers.add(inputFormat.createRecordReader(new FileSplit(path, 0, Long.MAX_VALUE, new String[]{}), new TaskAttemptContextImpl(configuration, new TaskAttemptID())));
- }
+ final Configuration configuration = ConfUtil.makeHadoopConfiguration(this.graph.configuration());
+ final InputFormat<NullWritable, VertexWritable> inputFormat = this.graph.configuration().getGraphInputFormat().getConstructor().newInstance();
+ if (inputFormat instanceof FileInputFormat) {
+ if (!this.graph.configuration().containsKey(Constants.GREMLIN_HADOOP_INPUT_LOCATION))
+ return; // there is not input location and thus, no data (empty graph)
+ configuration.set(Constants.MAPREDUCE_INPUT_FILEINPUTFORMAT_INPUTDIR, this.graph.configuration().getInputLocation());
+ }
+ final List<InputSplit> splits = inputFormat.getSplits(new JobContextImpl(configuration, new JobID(UUID.randomUUID().toString(), 1)));
+ for (final InputSplit split : splits) {
+ this.readers.add(inputFormat.createRecordReader(split, new TaskAttemptContextImpl(configuration, new TaskAttemptID())));
}
} catch (Exception e) {
throw new IllegalStateException(e.getMessage(), e);
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/85eb63db/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopVertexIterator.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopVertexIterator.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopVertexIterator.java
index 8977692..8f13c59 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopVertexIterator.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopVertexIterator.java
@@ -36,10 +36,6 @@ public final class HadoopVertexIterator extends HadoopElementIterator<Vertex> {
private HadoopVertex nextVertex = null;
- public HadoopVertexIterator(final HadoopGraph graph, final InputFormat<NullWritable, VertexWritable> inputFormat, final Path path) throws IOException, InterruptedException {
- super(graph, inputFormat, path);
- }
-
public HadoopVertexIterator(final HadoopGraph graph) throws IOException {
super(graph);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/85eb63db/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraphStructureStandardTest.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraphStructureStandardTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraphStructureStandardTest.java
new file mode 100644
index 0000000..459c900
--- /dev/null
+++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraphStructureStandardTest.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.hadoop.structure;
+
+import org.apache.tinkerpop.gremlin.GraphProviderClass;
+import org.apache.tinkerpop.gremlin.hadoop.HadoopGraphProvider;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.structure.StructureStandardSuite;
+import org.junit.runner.RunWith;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+@RunWith(StructureStandardSuite.class)
+@GraphProviderClass(provider = HadoopGraphProvider.class, graph = HadoopGraph.class)
+public class HadoopGraphStructureStandardTest {
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/85eb63db/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopGraphStructureStandardTest.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopGraphStructureStandardTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopGraphStructureStandardTest.java
deleted file mode 100644
index ccb2d64..0000000
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopGraphStructureStandardTest.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.structure.io;
-
-import org.apache.tinkerpop.gremlin.GraphProviderClass;
-import org.apache.tinkerpop.gremlin.hadoop.HadoopGraphProvider;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.structure.StructureStandardSuite;
-import org.junit.runner.RunWith;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-@RunWith(StructureStandardSuite.class)
-@GraphProviderClass(provider = HadoopGraphProvider.class, graph = HadoopGraph.class)
-public class HadoopGraphStructureStandardTest {
-}
-
-
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/85eb63db/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDFormat.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDFormat.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDFormat.java
new file mode 100644
index 0000000..3952c66
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDFormat.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.structure.io;
+
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.spark.SparkConf;
+import org.apache.spark.SparkContext;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
+import scala.Tuple2;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.UUID;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class InputRDDFormat extends InputFormat<NullWritable, VertexWritable> {
+
+ public InputRDDFormat() {
+
+ }
+
+ @Override
+ public List<InputSplit> getSplits(final JobContext jobContext) throws IOException, InterruptedException {
+ return Collections.singletonList(new InputSplit() {
+ @Override
+ public long getLength() throws IOException, InterruptedException {
+ return 0;
+ }
+
+ @Override
+ public String[] getLocations() throws IOException, InterruptedException {
+ return new String[0];
+ }
+ });
+ }
+
+ @Override
+ public RecordReader<NullWritable, VertexWritable> createRecordReader(final InputSplit inputSplit, final TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
+ try {
+ final org.apache.hadoop.conf.Configuration hadoopConfiguration = taskAttemptContext.getConfiguration();
+ final SparkConf sparkConfiguration = new SparkConf();
+ sparkConfiguration.setAppName(UUID.randomUUID().toString());
+ hadoopConfiguration.forEach(entry -> sparkConfiguration.set(entry.getKey(), entry.getValue()));
+ InputRDD inputRDD = (InputRDD) Class.forName(sparkConfiguration.get(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD)).newInstance();
+ JavaSparkContext javaSparkContext = new JavaSparkContext(SparkContext.getOrCreate(sparkConfiguration));
+ final Iterator<Tuple2<Object, VertexWritable>> iterator = inputRDD.readGraphRDD(ConfUtil.makeApacheConfiguration(taskAttemptContext.getConfiguration()), javaSparkContext).toLocalIterator();
+ return new RecordReader<NullWritable, VertexWritable>() {
+ @Override
+ public void initialize(final InputSplit inputSplit, final TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
+
+ }
+
+ @Override
+ public boolean nextKeyValue() throws IOException, InterruptedException {
+ return iterator.hasNext();
+ }
+
+ @Override
+ public NullWritable getCurrentKey() throws IOException, InterruptedException {
+ return NullWritable.get();
+ }
+
+ @Override
+ public VertexWritable getCurrentValue() throws IOException, InterruptedException {
+ return iterator.next()._2();
+ }
+
+ @Override
+ public float getProgress() throws IOException, InterruptedException {
+ return 1.0f; // TODO: make this dynamic (how? its an iterator.)
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (!hadoopConfiguration.getBoolean(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, false))
+ javaSparkContext.close();
+ }
+ };
+ } catch (final ClassNotFoundException | InstantiationException | IllegalAccessException e) {
+ throw new IOException(e.getMessage(), e);
+ }
+
+ }
+
+ /*private static class PartitionInputSplit extends InputSplit {
+
+ private final Partition partition;
+
+ public PartitionInputSplit(final Partition partition) {
+ this.partition = partition;
+ }
+
+ @Override
+ public long getLength() throws IOException, InterruptedException {
+ return 0;
+ }
+
+ @Override
+ public String[] getLocations() throws IOException, InterruptedException {
+ return new String[0];
+ }
+
+ public Partition getPartition() {
+ return this.partition;
+ }
+ }*/
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/85eb63db/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
index 5ba3b12..98a2b9f 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
@@ -24,6 +24,7 @@ import org.apache.spark.serializer.KryoSerializer;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
+import org.apache.tinkerpop.gremlin.process.traversal.P;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
import org.apache.tinkerpop.gremlin.structure.Graph;
@@ -52,4 +53,24 @@ public class InputRDDTest {
assertEquals(123l, graph.traversal(GraphTraversalSource.computer(SparkGraphComputer.class)).V().values("age").sum().next());
assertEquals(Long.valueOf(4l), graph.traversal(GraphTraversalSource.computer(SparkGraphComputer.class)).V().count().next());
}
+
+ @Test
+ public void shouldSupportHadoopGraphOLTP() {
+ final Configuration configuration = new BaseConfiguration();
+ configuration.setProperty("spark.master", "local[4]");
+ configuration.setProperty("spark.serializer", KryoSerializer.class.getCanonicalName());
+ configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
+ configuration.setProperty(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, ExampleInputRDD.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT, GryoOutputFormat.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, "target/test-output");
+ configuration.setProperty(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
+ ////////
+ Graph graph = GraphFactory.open(configuration);
+ GraphTraversalSource g = graph.traversal(); // OLTP;
+ assertEquals("person", g.V().has("age", 29).next().label());
+ assertEquals(Long.valueOf(4), g.V().count().next());
+ assertEquals(Long.valueOf(0), g.E().count().next());
+ assertEquals(Long.valueOf(2), g.V().has("age", P.gt(30)).count().next());
+ }
}
[16/31] incubator-tinkerpop git commit: Random exceptions started
popping up. I realized that I should kill any residual SparkContext before
and after every test. Added AbstractSparkTest which has a Before/After
annotated method which does the job. No ra
Posted by sp...@apache.org.
Random exceptions started popping up. I realized that I should kill any residual SparkContext before and after every test. Added AbstractSparkTest which has a Before/After annotated method which does the job. No random exceptions now. Thank god.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/87382a9e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/87382a9e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/87382a9e
Branch: refs/heads/TINKERPOP3-860
Commit: 87382a9e2945cac83e8e422a9525e599688e2c34
Parents: 5fda89a
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed Dec 2 18:35:58 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Dec 3 12:47:53 2015 -0700
----------------------------------------------------------------------
.../gremlin/spark/AbstractSparkTest.java | 43 ++++++++++++++++++++
.../process/computer/LocalPropertyTest.java | 4 +-
.../spark/structure/io/InputOutputRDDTest.java | 3 +-
.../spark/structure/io/InputRDDTest.java | 3 +-
.../spark/structure/io/OutputRDDTest.java | 3 +-
.../io/PersistedInputOutputRDDTest.java | 4 +-
6 files changed, 53 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/87382a9e/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/AbstractSparkTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/AbstractSparkTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/AbstractSparkTest.java
new file mode 100644
index 0000000..02b0e18
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/AbstractSparkTest.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark;
+
+import org.apache.spark.SparkConf;
+import org.apache.spark.SparkContext;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.junit.After;
+import org.junit.Before;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public abstract class AbstractSparkTest {
+
+ @After
+ @Before
+ public void setupTest() {
+ SparkConf sparkConfiguration = new SparkConf();
+ sparkConfiguration.setAppName(this.getClass().getCanonicalName() + "-setupTest");
+ sparkConfiguration.set("spark.master", "local[4]");
+ JavaSparkContext sparkContext = new JavaSparkContext(SparkContext.getOrCreate(sparkConfiguration));
+ sparkContext.close();
+ System.out.println("SparkContext has been closed for " + this.getClass().getCanonicalName() + "-setupTest");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/87382a9e/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/LocalPropertyTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/LocalPropertyTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/LocalPropertyTest.java
index 671bee8..b27b7b9 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/LocalPropertyTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/LocalPropertyTest.java
@@ -33,6 +33,7 @@ import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
+import org.apache.tinkerpop.gremlin.spark.AbstractSparkTest;
import org.apache.tinkerpop.gremlin.spark.structure.io.PersistedInputRDD;
import org.apache.tinkerpop.gremlin.spark.structure.io.PersistedOutputRDD;
import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
@@ -44,8 +45,7 @@ import java.util.UUID;
import static org.junit.Assert.assertTrue;
-public class LocalPropertyTest {
-
+public class LocalPropertyTest extends AbstractSparkTest {
@Test
public void shouldSetThreadLocalProperties() throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/87382a9e/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputRDDTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputRDDTest.java
index 50a43bc..cd78d7d 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputRDDTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputRDDTest.java
@@ -26,6 +26,7 @@ import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
+import org.apache.tinkerpop.gremlin.spark.AbstractSparkTest;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
import org.apache.tinkerpop.gremlin.structure.Graph;
@@ -35,7 +36,7 @@ import org.junit.Test;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-public class InputOutputRDDTest {
+public class InputOutputRDDTest extends AbstractSparkTest {
@Test
public void shouldReadFromWriteToArbitraryRDD() throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/87382a9e/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
index b64139c..1a54af0 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
@@ -25,6 +25,7 @@ import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
import org.apache.tinkerpop.gremlin.process.traversal.P;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.spark.AbstractSparkTest;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
import org.apache.tinkerpop.gremlin.structure.Graph;
@@ -36,7 +37,7 @@ import static org.junit.Assert.assertEquals;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-public class InputRDDTest {
+public class InputRDDTest extends AbstractSparkTest {
@Test
public void shouldReadFromArbitraryRDD() {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/87382a9e/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java
index 60790e7..2ccf0ca 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java
@@ -27,6 +27,7 @@ import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
+import org.apache.tinkerpop.gremlin.spark.AbstractSparkTest;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkHadoopGraphProvider;
import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
@@ -37,7 +38,7 @@ import org.junit.Test;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-public class OutputRDDTest {
+public class OutputRDDTest extends AbstractSparkTest {
@Test
public void shouldWriteToArbitraryRDD() throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/87382a9e/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java
index 1de2b47..bc55c04 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java
@@ -34,6 +34,7 @@ import org.apache.tinkerpop.gremlin.process.computer.ranking.pagerank.PageRankVe
import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
+import org.apache.tinkerpop.gremlin.spark.AbstractSparkTest;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkHadoopGraphProvider;
import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
@@ -50,10 +51,9 @@ import static org.junit.Assert.*;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-public class PersistedInputOutputRDDTest {
+public class PersistedInputOutputRDDTest extends AbstractSparkTest {
@Test
-
public void shouldNotPersistRDDAcrossJobs() throws Exception {
final String rddName = "target/test-output/" + UUID.randomUUID();
final Configuration configuration = new BaseConfiguration();
[20/31] incubator-tinkerpop git commit: GroovyProcessCompiuterSuite
was missing GroovyFlatMapTest. Added it. Added HadoopPool registration to
ToyGraphInputRDD so it doesn't give a WARN message. Also I tweaked
BulkLoaderVertexProgramTest to use target/tes
Posted by sp...@apache.org.
GroovyProcessCompiuterSuite was missing GroovyFlatMapTest. Added it. Added HadoopPool registration to ToyGraphInputRDD so it doesn't give a WARN message. Also I tweaked BulkLoaderVertexProgramTest to use target/test-output/ for its intermediary data.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/f8209353
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/f8209353
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/f8209353
Branch: refs/heads/TINKERPOP3-860
Commit: f820935303799b8dd661e4fa3686a24a36baad67
Parents: 26dcf3c
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed Dec 2 15:44:50 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Dec 3 12:47:53 2015 -0700
----------------------------------------------------------------------
.../process/GroovyProcessComputerSuite.java | 2 ++
.../BulkLoaderVertexProgramTest.java | 2 +-
.../computer/SparkHadoopGraphProvider.java | 16 +++++----
.../spark/structure/io/ToyGraphInputRDD.java | 38 +++++++++++---------
4 files changed, 33 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f8209353/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java
----------------------------------------------------------------------
diff --git a/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java b/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java
index 9df7902..d07d52f 100644
--- a/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java
+++ b/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java
@@ -47,6 +47,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyAddEdgeTest
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyCoalesceTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyConstantTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyCountTest;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyFlatMapTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyFoldTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyGraphTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyLoopsTest;
@@ -128,6 +129,7 @@ public class GroovyProcessComputerSuite extends ProcessComputerSuite {
GroovyCoalesceTest.Traversals.class,
GroovyConstantTest.Traversals.class,
GroovyCountTest.Traversals.class,
+ GroovyFlatMapTest.Traversals.class,
GroovyFoldTest.Traversals.class,
GroovyGraphTest.Traversals.class,
GroovyLoopsTest.Traversals.class,
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f8209353/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java
index 61a267f..992bfa4 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java
@@ -52,7 +52,7 @@ import static org.junit.Assert.assertTrue;
*/
public class BulkLoaderVertexProgramTest extends AbstractGremlinProcessTest {
- final static String TINKERGRAPH_LOCATION = "/tmp/tinkertest.kryo";
+ final static String TINKERGRAPH_LOCATION = "target/test-output/tinkertest.kryo";
private BulkLoader getBulkLoader(final BulkLoaderVertexProgram blvp) throws Exception {
final Field field = BulkLoaderVertexProgram.class.getDeclaredField("bulkLoader");
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f8209353/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
index 2cfeea3..de5a60d 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
@@ -22,6 +22,8 @@ import org.apache.tinkerpop.gremlin.GraphProvider;
import org.apache.tinkerpop.gremlin.LoadGraphWith;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.HadoopGraphProvider;
+import org.apache.tinkerpop.gremlin.process.computer.bulkloading.BulkLoaderVertexProgramTest;
+import org.apache.tinkerpop.gremlin.process.computer.ranking.PageRankVertexProgramTest;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
import org.apache.tinkerpop.gremlin.spark.structure.io.InputRDDFormat;
@@ -43,13 +45,13 @@ public final class SparkHadoopGraphProvider extends HadoopGraphProvider {
@Override
public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
final Map<String, Object> config = super.getBaseConfiguration(graphName, test, testMethodName, loadGraphWith);
- if (null != loadGraphWith) {
- if (RANDOM.nextBoolean()) {
- config.remove(Constants.GREMLIN_HADOOP_INPUT_LOCATION);
- config.put(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, ToyGraphInputRDD.class.getCanonicalName());
- config.put(ToyGraphInputRDD.GREMLIN_SPARK_TOY_GRAPH, loadGraphWith.toString());
- config.put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName());
- }
+ config.put(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true); // this makes the test suite go really fast
+ if (null != loadGraphWith &&
+ !test.equals(BulkLoaderVertexProgramTest.class) &&
+ !test.equals(PageRankVertexProgramTest.class) &&
+ RANDOM.nextBoolean()) {
+ config.put(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, ToyGraphInputRDD.class.getCanonicalName());
+ config.put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName());
}
/// spark configuration
config.put("spark.master", "local[4]");
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f8209353/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
index ed97c04..ea3636f 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
@@ -22,11 +22,13 @@ package org.apache.tinkerpop.gremlin.spark.structure.io;
import org.apache.commons.configuration.Configuration;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.tinkerpop.gremlin.LoadGraphWith;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPools;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.io.GraphReader;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
@@ -34,36 +36,38 @@ import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
import scala.Tuple2;
import java.io.IOException;
+import java.io.InputStream;
import java.util.List;
-import java.util.stream.Collectors;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
public final class ToyGraphInputRDD implements InputRDD {
- public static final String GREMLIN_SPARK_TOY_GRAPH = "gremlin.spark.toyGraph";
-
@Override
public JavaPairRDD<Object, VertexWritable> readGraphRDD(final Configuration configuration, final JavaSparkContext sparkContext) {
- final List<Vertex> vertices;
- if (configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH).equals(LoadGraphWith.GraphData.MODERN.toString()))
- vertices = IteratorUtils.list(TinkerFactory.createModern().vertices());
- else if (configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH).equals(LoadGraphWith.GraphData.CLASSIC.toString()))
- vertices = IteratorUtils.list(TinkerFactory.createClassic().vertices());
- else if (configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH).equals(LoadGraphWith.GraphData.CREW.toString()))
- vertices = IteratorUtils.list(TinkerFactory.createTheCrew().vertices());
- else if (configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH).equals(LoadGraphWith.GraphData.GRATEFUL.toString())) {
+ HadoopPools.initialize(TinkerGraph.open().configuration());
+ final List<VertexWritable> vertices;
+ if (configuration.getString(Constants.GREMLIN_HADOOP_INPUT_LOCATION).contains("modern"))
+ vertices = IteratorUtils.list(IteratorUtils.map(TinkerFactory.createModern().vertices(), VertexWritable::new));
+ else if (configuration.getString(Constants.GREMLIN_HADOOP_INPUT_LOCATION).contains("classic"))
+ vertices = IteratorUtils.list(IteratorUtils.map(TinkerFactory.createClassic().vertices(), VertexWritable::new));
+ else if (configuration.getString(Constants.GREMLIN_HADOOP_INPUT_LOCATION).contains("crew"))
+ vertices = IteratorUtils.list(IteratorUtils.map(TinkerFactory.createTheCrew().vertices(), VertexWritable::new));
+ else if (configuration.getString(Constants.GREMLIN_HADOOP_INPUT_LOCATION).contains("grateful")) {
try {
final Graph graph = TinkerGraph.open();
- graph.io(GryoIo.build()).readGraph(GryoResourceAccess.class.getResource("grateful-dead.kryo").getFile());
- vertices = IteratorUtils.list(graph.vertices());
+ final GraphReader reader = GryoReader.build().mapper(graph.io(GryoIo.build()).mapper().create()).create();
+ try (final InputStream stream = GryoResourceAccess.class.getResourceAsStream("grateful-dead.kryo")) {
+ reader.readGraph(stream, graph);
+ }
+ vertices = IteratorUtils.list(IteratorUtils.map(graph.vertices(), VertexWritable::new));
} catch (final IOException e) {
throw new IllegalStateException(e.getMessage(), e);
}
} else
- throw new IllegalArgumentException("No legal toy graph was provided to load: " + configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH));
+ throw new IllegalArgumentException("No legal toy graph was provided to load: " + configuration.getProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION));
- return sparkContext.parallelize(vertices.stream().map(VertexWritable::new).collect(Collectors.toList())).mapToPair(vertex -> new Tuple2<>(vertex.get().id(), vertex));
+ return sparkContext.parallelize(vertices).mapToPair(vertex -> new Tuple2<>(vertex.get().id(), vertex));
}
}
[11/31] incubator-tinkerpop git commit: I have the
SparkIntegrationTestSuite now testing either from Gryo FileInputFormat,
GraphSON FileInputFormat,
or an InputRDD. This gives us super coverage and proves that InputRDD
(bypassing Hadoop) is working as ex
Posted by sp...@apache.org.
I have the SparkIntegrationTestSuite now testing either from Gryo FileInputFormat, GraphSON FileInputFormat, or an InputRDD. This gives us super coverage and proves that InputRDD (bypassing Hadoop) is working as expected. I also fixed up some other tests that used KryoSerializer instead of GryoSerializer as I learned how to deal with Scalas WrappedArray class. It was insane. This is really good stuff.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/ddbbec08
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/ddbbec08
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/ddbbec08
Branch: refs/heads/TINKERPOP3-860
Commit: ddbbec084635a7e764632713a07b0c532a614e4c
Parents: 85eb63d
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed Dec 2 12:02:09 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Dec 3 12:47:52 2015 -0700
----------------------------------------------------------------------
.../structure/hdfs/HadoopElementIterator.java | 6 ++-
.../computer/payload/MessagePayload.java | 5 +-
.../computer/payload/ViewIncomingPayload.java | 6 +--
.../computer/payload/ViewOutgoingPayload.java | 12 +++--
.../spark/structure/io/gryo/GryoSerializer.java | 6 +++
.../io/gryo/WrappedArraySerializer.java | 46 ++++++++++++++++++
.../computer/SparkHadoopGraphProvider.java | 40 ++++++++++++++--
.../spark/structure/io/ClassicInputRDD.java | 39 +++++++++++++++
.../spark/structure/io/GratefulInputRDD.java | 50 ++++++++++++++++++++
.../spark/structure/io/InputOutputRDDTest.java | 3 +-
.../spark/structure/io/InputRDDTest.java | 5 +-
.../spark/structure/io/ModernInputRDD.java | 41 ++++++++++++++++
.../spark/structure/io/OutputRDDTest.java | 3 +-
.../spark/structure/io/TheCrewInputRDD.java | 39 +++++++++++++++
14 files changed, 285 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ddbbec08/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopElementIterator.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopElementIterator.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopElementIterator.java
index f9ffea2..45f3c55 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopElementIterator.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/hdfs/HadoopElementIterator.java
@@ -19,6 +19,8 @@
package org.apache.tinkerpop.gremlin.hadoop.structure.hdfs;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.InputSplit;
@@ -56,7 +58,9 @@ public abstract class HadoopElementIterator<E extends Element> implements Iterat
final InputFormat<NullWritable, VertexWritable> inputFormat = this.graph.configuration().getGraphInputFormat().getConstructor().newInstance();
if (inputFormat instanceof FileInputFormat) {
if (!this.graph.configuration().containsKey(Constants.GREMLIN_HADOOP_INPUT_LOCATION))
- return; // there is not input location and thus, no data (empty graph)
+ return; // there is no input location and thus, no data (empty graph)
+ if (!FileSystem.get(configuration).exists(new Path(this.graph.configuration().getInputLocation())))
+ return; // there is no data at the input location (empty graph)
configuration.set(Constants.MAPREDUCE_INPUT_FILEINPUTFORMAT_INPUTDIR, this.graph.configuration().getInputLocation());
}
final List<InputSplit> splits = inputFormat.getSplits(new JobContextImpl(configuration, new JobID(UUID.randomUUID().toString(), 1)));
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ddbbec08/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/MessagePayload.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/MessagePayload.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/MessagePayload.java
index 09e2599..f32ec44 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/MessagePayload.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/MessagePayload.java
@@ -23,7 +23,10 @@ package org.apache.tinkerpop.gremlin.spark.process.computer.payload;
*/
public final class MessagePayload<M> implements Payload {
- private final M message;
+ private M message;
+
+ private MessagePayload() {
+ }
public MessagePayload(final M message) {
this.message = message;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ddbbec08/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewIncomingPayload.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewIncomingPayload.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewIncomingPayload.java
index 911fc7b..a2c9205 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewIncomingPayload.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewIncomingPayload.java
@@ -31,11 +31,11 @@ import java.util.List;
public final class ViewIncomingPayload<M> implements Payload {
private List<DetachedVertexProperty<Object>> view = null;
- private final List<M> incomingMessages;
+ private List<M> incomingMessages;
- public ViewIncomingPayload() {
- this.incomingMessages = null;
+ private ViewIncomingPayload() {
+
}
public ViewIncomingPayload(final MessageCombiner<M> messageCombiner) {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ddbbec08/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewOutgoingPayload.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewOutgoingPayload.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewOutgoingPayload.java
index fc4aeed..20c8e09 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewOutgoingPayload.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewOutgoingPayload.java
@@ -28,10 +28,14 @@ import java.util.List;
*/
public final class ViewOutgoingPayload<M> implements Payload {
- private final List<DetachedVertexProperty<Object>> view;
- private final List<Tuple2<Object,M>> outgoingMessages;
+ private List<DetachedVertexProperty<Object>> view;
+ private List<Tuple2<Object, M>> outgoingMessages;
- public ViewOutgoingPayload(final List<DetachedVertexProperty<Object>> view, final List<Tuple2<Object,M>> outgoingMessages) {
+ private ViewOutgoingPayload() {
+
+ }
+
+ public ViewOutgoingPayload(final List<DetachedVertexProperty<Object>> view, final List<Tuple2<Object, M>> outgoingMessages) {
this.view = view;
this.outgoingMessages = outgoingMessages;
}
@@ -40,7 +44,7 @@ public final class ViewOutgoingPayload<M> implements Payload {
return new ViewPayload(this.view);
}
- public List<Tuple2<Object,M>> getOutgoingMessages() {
+ public List<Tuple2<Object, M>> getOutgoingMessages() {
return this.outgoingMessages;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ddbbec08/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java
index 7202892..29fde9f 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java
@@ -41,6 +41,8 @@ import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoPool;
import org.apache.tinkerpop.shaded.kryo.io.Output;
import org.apache.tinkerpop.shaded.kryo.serializers.JavaSerializer;
import scala.Tuple2;
+import scala.Tuple3;
+import scala.collection.mutable.WrappedArray;
import scala.runtime.BoxedUnit;
import java.util.Collections;
@@ -79,11 +81,15 @@ public final class GryoSerializer extends Serializer {
try {
builder.addCustom(SerializableWritable.class, new JavaSerializer())
.addCustom(Tuple2.class, new JavaSerializer())
+ .addCustom(Tuple2[].class, new JavaSerializer())
+ .addCustom(Tuple3.class, new JavaSerializer())
+ .addCustom(Tuple3[].class, new JavaSerializer())
.addCustom(CompressedMapStatus.class, new JavaSerializer())
.addCustom(HttpBroadcast.class, new JavaSerializer())
.addCustom(PythonBroadcast.class, new JavaSerializer())
.addCustom(BoxedUnit.class, new JavaSerializer())
.addCustom(Class.forName("scala.reflect.ClassTag$$anon$1"), new JavaSerializer())
+ .addCustom(WrappedArray.ofRef.class, new WrappedArraySerializer())
.addCustom(MessagePayload.class)
.addCustom(ViewIncomingPayload.class)
.addCustom(ViewOutgoingPayload.class)
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ddbbec08/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/WrappedArraySerializer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/WrappedArraySerializer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/WrappedArraySerializer.java
new file mode 100644
index 0000000..0e9f03f
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/WrappedArraySerializer.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.structure.io.gryo;
+
+import org.apache.tinkerpop.shaded.kryo.Kryo;
+import org.apache.tinkerpop.shaded.kryo.Serializer;
+import org.apache.tinkerpop.shaded.kryo.io.Input;
+import org.apache.tinkerpop.shaded.kryo.io.Output;
+import scala.collection.JavaConversions;
+import scala.collection.mutable.WrappedArray;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class WrappedArraySerializer<T> extends Serializer<WrappedArray<T>> {
+
+ @Override
+ public void write(final Kryo kryo, final Output output, final WrappedArray<T> iterable) {
+ kryo.writeClassAndObject(output,new ArrayList<>(JavaConversions.asJavaList(iterable)));
+ }
+
+ @Override
+ public WrappedArray<T> read(final Kryo kryo, final Input input, final Class<WrappedArray<T>> aClass) {
+ return new WrappedArray.ofRef<>((T[]) ((List<T>) kryo.readClassAndObject(input)).toArray());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ddbbec08/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
index c81ea92..2328176 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
@@ -20,12 +20,19 @@ package org.apache.tinkerpop.gremlin.spark.process.computer;
import org.apache.tinkerpop.gremlin.GraphProvider;
import org.apache.tinkerpop.gremlin.LoadGraphWith;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.HadoopGraphProvider;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
+import org.apache.tinkerpop.gremlin.spark.structure.io.ClassicInputRDD;
+import org.apache.tinkerpop.gremlin.spark.structure.io.GratefulInputRDD;
+import org.apache.tinkerpop.gremlin.spark.structure.io.InputRDDFormat;
+import org.apache.tinkerpop.gremlin.spark.structure.io.ModernInputRDD;
+import org.apache.tinkerpop.gremlin.spark.structure.io.TheCrewInputRDD;
import org.apache.tinkerpop.gremlin.structure.Graph;
import java.util.Map;
+import java.util.Random;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
@@ -33,13 +40,41 @@ import java.util.Map;
@GraphProvider.Descriptor(computer = SparkGraphComputer.class)
public final class SparkHadoopGraphProvider extends HadoopGraphProvider {
+ private static final Random RANDOM = new Random();
+
@Override
public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
final Map<String, Object> config = super.getBaseConfiguration(graphName, test, testMethodName, loadGraphWith);
- config.put("mapreduce.job.reduces", 4);
+ if (null != loadGraphWith) {
+ if (loadGraphWith.equals(LoadGraphWith.GraphData.MODERN)) {
+ if (RANDOM.nextBoolean()) {
+ config.remove(Constants.GREMLIN_HADOOP_INPUT_LOCATION);
+ config.put(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, ModernInputRDD.class.getCanonicalName());
+ config.put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName());
+ }
+ } else if (loadGraphWith.equals(LoadGraphWith.GraphData.CREW)) {
+ if (RANDOM.nextBoolean()) {
+ config.remove(Constants.GREMLIN_HADOOP_INPUT_LOCATION);
+ config.put(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, TheCrewInputRDD.class.getCanonicalName());
+ config.put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName());
+ }
+ } else if (loadGraphWith.equals(LoadGraphWith.GraphData.CLASSIC)) {
+ if (RANDOM.nextBoolean()) {
+ config.remove(Constants.GREMLIN_HADOOP_INPUT_LOCATION);
+ config.put(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, ClassicInputRDD.class.getCanonicalName());
+ config.put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName());
+ }
+ } else if (loadGraphWith.equals(LoadGraphWith.GraphData.GRATEFUL)) {
+ if (RANDOM.nextBoolean()) {
+ config.remove(Constants.GREMLIN_HADOOP_INPUT_LOCATION);
+ config.put(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, GratefulInputRDD.class.getCanonicalName());
+ config.put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName());
+ }
+ }
+ }
/// spark configuration
config.put("spark.master", "local[4]");
- // put("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
+ //config.put("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
config.put("spark.serializer", "org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer");
config.put("spark.kryo.registrationRequired", true);
return config;
@@ -49,5 +84,4 @@ public final class SparkHadoopGraphProvider extends HadoopGraphProvider {
public GraphTraversalSource traversal(final Graph graph) {
return GraphTraversalSource.build().engine(ComputerTraversalEngine.build().computer(SparkGraphComputer.class)).create(graph);
}
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ddbbec08/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ClassicInputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ClassicInputRDD.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ClassicInputRDD.java
new file mode 100644
index 0000000..4512b61
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ClassicInputRDD.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.structure.io;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+import scala.Tuple2;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class ClassicInputRDD implements InputRDD {
+
+ @Override
+ public JavaPairRDD<Object, VertexWritable> readGraphRDD(final Configuration configuration, final JavaSparkContext sparkContext) {
+ return sparkContext.parallelize(IteratorUtils.list(IteratorUtils.map(TinkerFactory.createClassic().vertices(), VertexWritable::new))).mapToPair(vertex -> new Tuple2<>(vertex.get().id(), vertex));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ddbbec08/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/GratefulInputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/GratefulInputRDD.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/GratefulInputRDD.java
new file mode 100644
index 0000000..396aa75
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/GratefulInputRDD.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.structure.io;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+import scala.Tuple2;
+
+import java.io.IOException;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class GratefulInputRDD implements InputRDD {
+
+ @Override
+ public JavaPairRDD<Object, VertexWritable> readGraphRDD(final Configuration configuration, final JavaSparkContext sparkContext) {
+ try {
+ final Graph graph = TinkerGraph.open();
+ graph.io(GryoIo.build()).readGraph(GryoResourceAccess.class.getResource("grateful-dead.kryo").getFile());
+ return sparkContext.parallelize(IteratorUtils.list(IteratorUtils.map(graph.vertices(), VertexWritable::new))).mapToPair(vertex -> new Tuple2<>(vertex.get().id(), vertex));
+ } catch (final IOException e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ddbbec08/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputRDDTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputRDDTest.java
index 3691aba..ea62114 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputRDDTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputRDDTest.java
@@ -28,6 +28,7 @@ import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexPr
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
+import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
import org.junit.Test;
@@ -41,7 +42,7 @@ public class InputOutputRDDTest {
public void shouldReadFromWriteToArbitraryRDD() throws Exception {
final Configuration configuration = new BaseConfiguration();
configuration.setProperty("spark.master", "local[4]");
- configuration.setProperty("spark.serializer", KryoSerializer.class.getCanonicalName());
+ configuration.setProperty("spark.serializer", GryoSerializer.class.getCanonicalName());
configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
configuration.setProperty(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, ExampleInputRDD.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_SPARK_GRAPH_OUTPUT_RDD, ExampleOutputRDD.class.getCanonicalName());
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ddbbec08/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
index 98a2b9f..2cbfd66 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
@@ -27,6 +27,7 @@ import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
import org.apache.tinkerpop.gremlin.process.traversal.P;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
+import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
import org.junit.Test;
@@ -42,7 +43,7 @@ public class InputRDDTest {
public void shouldReadFromArbitraryRDD() {
final Configuration configuration = new BaseConfiguration();
configuration.setProperty("spark.master", "local[4]");
- configuration.setProperty("spark.serializer", KryoSerializer.class.getCanonicalName());
+ configuration.setProperty("spark.serializer", GryoSerializer.class.getCanonicalName());
configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
configuration.setProperty(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, ExampleInputRDD.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT, GryoOutputFormat.class.getCanonicalName());
@@ -58,7 +59,7 @@ public class InputRDDTest {
public void shouldSupportHadoopGraphOLTP() {
final Configuration configuration = new BaseConfiguration();
configuration.setProperty("spark.master", "local[4]");
- configuration.setProperty("spark.serializer", KryoSerializer.class.getCanonicalName());
+ configuration.setProperty("spark.serializer", GryoSerializer.class.getCanonicalName());
configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
configuration.setProperty(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, ExampleInputRDD.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName());
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ddbbec08/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ModernInputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ModernInputRDD.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ModernInputRDD.java
new file mode 100644
index 0000000..849e3e6
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ModernInputRDD.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.structure.io;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+import scala.Tuple2;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class ModernInputRDD implements InputRDD {
+
+ @Override
+ public JavaPairRDD<Object, VertexWritable> readGraphRDD(final Configuration configuration, final JavaSparkContext sparkContext) {
+ return sparkContext.
+ parallelize(IteratorUtils.list(IteratorUtils.map(TinkerFactory.createModern().vertices(), VertexWritable::new))).
+ mapToPair(vertex -> new Tuple2<>(vertex.get().id(), vertex));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ddbbec08/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java
index 10eecb3..f9b6f39 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java
@@ -30,6 +30,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSo
import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkHadoopGraphProvider;
+import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
import org.junit.Test;
@@ -43,7 +44,7 @@ public class OutputRDDTest {
public void shouldWriteToArbitraryRDD() throws Exception {
final Configuration configuration = new BaseConfiguration();
configuration.setProperty("spark.master", "local[4]");
- configuration.setProperty("spark.serializer", KryoSerializer.class.getCanonicalName());
+ configuration.setProperty("spark.serializer", GryoSerializer.class.getCanonicalName());
configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, GryoInputFormat.class.getCanonicalName());
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ddbbec08/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/TheCrewInputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/TheCrewInputRDD.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/TheCrewInputRDD.java
new file mode 100644
index 0000000..ff5a274
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/TheCrewInputRDD.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.structure.io;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+import scala.Tuple2;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class TheCrewInputRDD implements InputRDD {
+
+ @Override
+ public JavaPairRDD<Object, VertexWritable> readGraphRDD(final Configuration configuration, final JavaSparkContext sparkContext) {
+ return sparkContext.parallelize(IteratorUtils.list(IteratorUtils.map(TinkerFactory.createTheCrew().vertices(), VertexWritable::new))).mapToPair(vertex -> new Tuple2<>(vertex.get().id(), vertex));
+ }
+}
[12/31] incubator-tinkerpop git commit: this is a breaking push. need
to get Groovy GremlinLoader happy about comparator. Will get back to this.
Posted by sp...@apache.org.
this is a breaking push. need to get Groovy GremlinLoader happy about comparator. Will get back to this.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/5b9de833
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/5b9de833
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/5b9de833
Branch: refs/heads/TINKERPOP3-860
Commit: 5b9de8333286cdf32d00b0f706b0f6f21600f30e
Parents: b7c6041
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon Nov 30 12:45:53 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Dec 3 12:47:52 2015 -0700
----------------------------------------------------------------------
.../java/org/apache/tinkerpop/gremlin/structure/Column.java | 2 +-
.../process/traversal/step/branch/GroovyLocalTest.groovy | 2 +-
.../process/traversal/step/map/GroovyOrderTest.groovy | 8 ++++----
.../gremlin/process/traversal/step/map/OrderTest.java | 8 ++++----
4 files changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/5b9de833/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Column.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Column.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Column.java
index fca90d7..38921c1 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Column.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Column.java
@@ -25,7 +25,7 @@ import java.util.Map;
import java.util.function.Function;
/**
- * Column references a particular type of column in a complex data structure such as a {@link java.util.Map} or a {@link org.apache.tinkerpop.gremlin.process.traversal.Path}.
+ * Column references a particular type of column in a complex data structure such as a {@link java.util.Map}, a {@link java.util.Map.Entry}, or a {@link org.apache.tinkerpop.gremlin.process.traversal.Path}.
*
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/5b9de833/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/branch/GroovyLocalTest.groovy
----------------------------------------------------------------------
diff --git a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/branch/GroovyLocalTest.groovy b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/branch/GroovyLocalTest.groovy
index 7ed2673..fe8d629 100644
--- a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/branch/GroovyLocalTest.groovy
+++ b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/branch/GroovyLocalTest.groovy
@@ -32,7 +32,7 @@ public abstract class GroovyLocalTest {
@Override
public Traversal<Vertex, String> get_g_V_localXpropertiesXlocationX_order_byXvalueX_limitX2XX_value() {
- TraversalScriptHelper.compute("g.V.local(__.properties('location').order.by(T.value,Order.incr).limit(2)).value", g);
+ TraversalScriptHelper.compute("g.V.local(__.properties('location').order.by(value,incr).limit(2)).value", g);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/5b9de833/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyOrderTest.groovy
----------------------------------------------------------------------
diff --git a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyOrderTest.groovy b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyOrderTest.groovy
index e9463bc..002a4e1 100644
--- a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyOrderTest.groovy
+++ b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyOrderTest.groovy
@@ -41,12 +41,12 @@ public abstract class GroovyOrderTest {
@Override
public Traversal<Vertex, String> get_g_V_order_byXname_incrX_name() {
- TraversalScriptHelper.compute("g.V.order.by('name', Order.incr).name", g)
+ TraversalScriptHelper.compute("g.V.order.by('name', incr).name", g)
}
@Override
public Traversal<Vertex, String> get_g_V_order_byXnameX_name() {
- TraversalScriptHelper.compute("g.V.order.by('name', Order.incr).name", g)
+ TraversalScriptHelper.compute("g.V.order.by('name', incr).name", g)
}
@Override
@@ -65,7 +65,7 @@ public abstract class GroovyOrderTest {
}
@Override
- public Traversal<Vertex, Map<Integer, Integer>> get_g_VX1X_hasXlabel_personX_mapXmapXint_ageXX_orderXlocalX_byXvalueDecrX_byXkeyIncrX(
+ public Traversal<Vertex, Map<Integer, Integer>> get_g_VX1X_hasXlabel_personX_mapXmapXint_ageXX_orderXlocalX_byXvalues_decrX_byXkeys_incrX(
final Object v1Id) {
TraversalScriptHelper.compute("""g.V(v1Id).map {
final Map map = [:];
@@ -74,7 +74,7 @@ public abstract class GroovyOrderTest {
map[3] = it.age * 3;
map[4] = it.age;
return map;
- }.order(local).by(valueDecr).by(keyIncr)""", g, "v1Id", v1Id);
+ }.order(local).by(values,decr).by(keys,incr)""", g, "v1Id", v1Id);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/5b9de833/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderTest.java
index 3f08d60..4475b7c 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderTest.java
@@ -65,7 +65,7 @@ public abstract class OrderTest extends AbstractGremlinProcessTest {
public abstract Traversal<Vertex, Map<String, Vertex>> get_g_V_asXaX_outXcreatedX_asXbX_order_byXshuffleX_selectXa_bX();
- public abstract Traversal<Vertex, Map<Integer, Integer>> get_g_VX1X_hasXlabel_personX_mapXmapXint_ageXX_orderXlocalX_byXvalueDecrX_byXkeyIncrX(final Object v1Id);
+ public abstract Traversal<Vertex, Map<Integer, Integer>> get_g_VX1X_hasXlabel_personX_mapXmapXint_ageXX_orderXlocalX_byXvalues_decrX_byXkeys_incrX(final Object v1Id);
public abstract Traversal<Vertex, Vertex> get_g_V_order_byXoutE_count__decrX();
@@ -196,8 +196,8 @@ public abstract class OrderTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(MODERN)
- public void g_VX1X_hasXlabel_personX_mapXmapXint_ageXX_orderXlocalX_byXvalueDecrX_byXkeyIncrX() {
- final Traversal<Vertex, Map<Integer, Integer>> traversal = get_g_VX1X_hasXlabel_personX_mapXmapXint_ageXX_orderXlocalX_byXvalueDecrX_byXkeyIncrX(convertToVertexId("marko"));
+ public void g_VX1X_hasXlabel_personX_mapXmapXint_ageXX_orderXlocalX_byXvalues_decrX_byXkeys_incrX() {
+ final Traversal<Vertex, Map<Integer, Integer>> traversal = get_g_VX1X_hasXlabel_personX_mapXmapXint_ageXX_orderXlocalX_byXvalues_decrX_byXkeys_incrX(convertToVertexId("marko"));
printTraversalForm(traversal);
final Map<Integer, Integer> map = traversal.next();
assertFalse(traversal.hasNext());
@@ -356,7 +356,7 @@ public abstract class OrderTest extends AbstractGremlinProcessTest {
}
@Override
- public Traversal<Vertex, Map<Integer, Integer>> get_g_VX1X_hasXlabel_personX_mapXmapXint_ageXX_orderXlocalX_byXvalueDecrX_byXkeyIncrX(final Object v1Id) {
+ public Traversal<Vertex, Map<Integer, Integer>> get_g_VX1X_hasXlabel_personX_mapXmapXint_ageXX_orderXlocalX_byXvalues_decrX_byXkeys_incrX(final Object v1Id) {
return g.V(v1Id).map(v -> {
final Map<Integer, Integer> map = new HashMap<>();
map.put(1, (int) v.get().value("age"));
[21/31] incubator-tinkerpop git commit: Introduce the "ci" profile to
Travis
Posted by sp...@apache.org.
Introduce the "ci" profile to Travis
It builds locally with a sizeable reduction in logging.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/b2fee755
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/b2fee755
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/b2fee755
Branch: refs/heads/TINKERPOP3-860
Commit: b2fee7559d5288779ec622f48b54724b52503eb2
Parents: 4795042
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Thu Dec 3 13:26:07 2015 -0500
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Dec 3 12:48:58 2015 -0700
----------------------------------------------------------------------
.travis.yml | 3 ++-
pom.xml | 10 ++--------
2 files changed, 4 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b2fee755/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index a8f56e5..659b57c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,4 +9,5 @@ addons:
packages:
- oracle-java8-installer
script:
- - "mvn clean install"
\ No newline at end of file
+ - "mvn clean install -Dci"
+
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b2fee755/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index c184a75..0332cdc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -133,7 +133,7 @@ limitations under the License.
the modules under the root directory, not the root
directory itself. -->
<log4j.properties.dir>${project.basedir}/../gremlin-server/conf/</log4j.properties.dir>
- <log4j.properties>log4j-server.properties</log4j.properties>
+ <log4j.properties>log4j.properties</log4j.properties>
<metrics.version>3.0.2</metrics.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
@@ -930,15 +930,9 @@ limitations under the License.
<artifactId>maven-surefire-plugin</artifactId>
<version>2.17</version>
<configuration>
- <argLine>-Dlog4j.configuration=${log4j.properties}
- </argLine>
<systemPropertyVariables>
- <muteTestLogs>false</muteTestLogs>
+ <muteTestLogs>true</muteTestLogs>
</systemPropertyVariables>
- <additionalClasspathElements>
- <!-- So that Log4j can find the Log4j properties file -->
- <additionalClasspathElement>${log4j.properties.dir}</additionalClasspathElement>
- </additionalClasspathElements>
<excludes>
<exclude>**/*IntegrateTest.java</exclude>
<exclude>**/*PerformanceTest.java</exclude>
[29/31] incubator-tinkerpop git commit: Improved the ability to
control logging for maven builds.
Posted by sp...@apache.org.
Improved the ability to control logging for maven builds.
Added a "silent" configuration for the "ci" profile so that travis will be happy.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/2b2bf960
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/2b2bf960
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/2b2bf960
Branch: refs/heads/TINKERPOP3-860
Commit: 2b2bf960f09e704244f91801116c081368fbdb11
Parents: 19beece
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Dec 4 09:08:00 2015 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Fri Dec 4 09:08:00 2015 -0500
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
docs/src/dev/developer/contributing.asciidoc | 13 +-
giraph-gremlin/pom.xml | 18 +-
.../src/test/resources/log4j-silent.properties | 18 ++
.../src/test/resources/log4j-test.properties | 23 ++
gremlin-console/pom.xml | 6 +
.../src/test/resources/log4j-silent.properties | 18 ++
.../src/test/resources/log4j-test.properties | 21 ++
gremlin-core/pom.xml | 6 +
.../src/test/resources/log4j-silent.properties | 18 ++
.../src/test/resources/log4j-test.properties | 21 ++
.../src/test/resources/log4j-silent.properties | 18 ++
.../src/test/resources/log4j-test.properties | 21 ++
.../src/test/resources/log4j-silent.properties | 18 ++
.../src/test/resources/log4j-test.properties | 21 ++
.../GremlinServerAuthIntegrateOldTest.java | 235 -------------------
.../GremlinServerAuthOldIntegrateTest.java | 235 +++++++++++++++++++
.../src/test/resources/log4j-silent.properties | 18 ++
.../src/test/resources/log4j-test.properties | 21 ++
gremlin-test/pom.xml | 6 +
.../tinkerpop/gremlin/AbstractGremlinTest.java | 6 +
.../src/test/resources/log4j-silent.properties | 21 ++
.../src/test/resources/log4j-test.properties | 24 ++
hadoop-gremlin/pom.xml | 18 +-
.../groovy/plugin/HadoopGremlinPluginCheck.java | 176 ++++++++++++++
.../groovy/plugin/HadoopGremlinPluginTest.java | 173 --------------
.../hadoop/groovy/plugin/HadoopPluginSuite.java | 2 +-
.../src/test/resources/log4j-silent.properties | 18 ++
.../src/test/resources/log4j-test.properties | 21 ++
neo4j-gremlin/pom.xml | 6 +
.../src/test/resources/log4j-silent.properties | 18 ++
.../src/test/resources/log4j-test.properties | 21 ++
pom.xml | 47 ++--
spark-gremlin/pom.xml | 18 +-
.../src/test/resources/log4j-silent.properties | 18 ++
.../src/test/resources/log4j-test.properties | 25 ++
.../src/test/resources/log4j-silent.properties | 18 ++
.../src/test/resources/log4j-test.properties | 21 ++
38 files changed, 932 insertions(+), 475 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 717aa71..1fa03d5 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,6 +26,7 @@ image::https://raw.githubusercontent.com/apache/incubator-tinkerpop/master/docs/
TinkerPop 3.1.1 (NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* Improved logging control during builds with Maven.
* `Column` now implements `Function` and now `by(valueDecr)` can be replaced with `by(values,decr)` and thus, projection and order are now separated.
* Added `InputRDDFormat` which wraps an `InputRDD` to make it accessible to Hadoop and not just Spark.
* Added `AbstractSparkTest` which handles closing `SparkContext` instances between tests now that we support persisted contexts.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/docs/src/dev/developer/contributing.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/dev/developer/contributing.asciidoc b/docs/src/dev/developer/contributing.asciidoc
index 70c78e2..959f4ef 100644
--- a/docs/src/dev/developer/contributing.asciidoc
+++ b/docs/src/dev/developer/contributing.asciidoc
@@ -401,4 +401,15 @@ the existing `<execution>` entries, paying special attention to the pathing of t
uploaded to the server and should preserve the directory structure in git as referenced in `<sourceDirectory>`.
Please see the <<building-testing,Building and Testing>> section for more information on how to generate the
-documentation.
\ No newline at end of file
+documentation.
+
+[[logging]]
+Logging
+~~~~~~~
+
+TinkerPop uses SLF4j for logging and typically leans back on Log4j as the implementation. Configuring log outputs
+for debugging purposes within tests can be altered by editing the `log4j-test.properties` file in each module's test
+resources. That file gets copied to the `target/test-classes` on build and surefire and failsafe plugins in maven
+are then configured to point at that area of the file system for those configuration files. The properties files
+can be edited to fine tune control of the log output, but generally speaking the current configuration is likely
+best for everyone's general purposes, so if changes are made please revert them prior to commit.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/giraph-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/giraph-gremlin/pom.xml b/giraph-gremlin/pom.xml
index 7a65788..ba037cf 100644
--- a/giraph-gremlin/pom.xml
+++ b/giraph-gremlin/pom.xml
@@ -203,6 +203,10 @@ limitations under the License.
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
</plugin>
<plugin>
@@ -240,20 +244,6 @@ limitations under the License.
</archive>
</configuration>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <!-- log4j.configuration>log4j-core.properties</log4j.configuration> -->
- <!--<argLine>-Xmx2048M</argLine>-->
- <excludes>
- <exclude>**/*IntegrateTest.java</exclude>
- <exclude>**/*PerformanceTest.java</exclude>
- <!-- this is technically a member of the integration test suite -->
- <exclude>**/HadoopGremlinPluginTest.java</exclude>
- </excludes>
- </configuration>
- </plugin>
</plugins>
</build>
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/giraph-gremlin/src/test/resources/log4j-silent.properties
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/test/resources/log4j-silent.properties b/giraph-gremlin/src/test/resources/log4j-silent.properties
new file mode 100644
index 0000000..9c6148d
--- /dev/null
+++ b/giraph-gremlin/src/test/resources/log4j-silent.properties
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+log4j.rootLogger=OFF
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/giraph-gremlin/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/test/resources/log4j-test.properties b/giraph-gremlin/src/test/resources/log4j-test.properties
new file mode 100644
index 0000000..a3e679c
--- /dev/null
+++ b/giraph-gremlin/src/test/resources/log4j-test.properties
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+log4j.rootLogger=WARN, stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n
+
+log4j.logger.org.apache.hadoop=ERROR
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/gremlin-console/pom.xml
----------------------------------------------------------------------
diff --git a/gremlin-console/pom.xml b/gremlin-console/pom.xml
index dfa9415..419fd7e 100644
--- a/gremlin-console/pom.xml
+++ b/gremlin-console/pom.xml
@@ -144,6 +144,12 @@ limitations under the License.
</directory>
</resource>
</resources>
+ <testResources>
+ <testResource>
+ <directory>${basedir}/src/test/resources
+ </directory>
+ </testResource>
+ </testResources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/gremlin-console/src/test/resources/log4j-silent.properties
----------------------------------------------------------------------
diff --git a/gremlin-console/src/test/resources/log4j-silent.properties b/gremlin-console/src/test/resources/log4j-silent.properties
new file mode 100644
index 0000000..9c6148d
--- /dev/null
+++ b/gremlin-console/src/test/resources/log4j-silent.properties
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+log4j.rootLogger=OFF
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/gremlin-console/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/gremlin-console/src/test/resources/log4j-test.properties b/gremlin-console/src/test/resources/log4j-test.properties
new file mode 100644
index 0000000..ef436fe
--- /dev/null
+++ b/gremlin-console/src/test/resources/log4j-test.properties
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+log4j.rootLogger=WARN, stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/gremlin-core/pom.xml
----------------------------------------------------------------------
diff --git a/gremlin-core/pom.xml b/gremlin-core/pom.xml
index 77d826e..668830a 100644
--- a/gremlin-core/pom.xml
+++ b/gremlin-core/pom.xml
@@ -102,6 +102,12 @@ limitations under the License.
<build>
<directory>${basedir}/target</directory>
<finalName>${project.artifactId}-${project.version}</finalName>
+ <testResources>
+ <testResource>
+ <directory>${basedir}/src/test/resources
+ </directory>
+ </testResource>
+ </testResources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/gremlin-core/src/test/resources/log4j-silent.properties
----------------------------------------------------------------------
diff --git a/gremlin-core/src/test/resources/log4j-silent.properties b/gremlin-core/src/test/resources/log4j-silent.properties
new file mode 100644
index 0000000..9c6148d
--- /dev/null
+++ b/gremlin-core/src/test/resources/log4j-silent.properties
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+log4j.rootLogger=OFF
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/gremlin-core/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/gremlin-core/src/test/resources/log4j-test.properties b/gremlin-core/src/test/resources/log4j-test.properties
new file mode 100644
index 0000000..ef436fe
--- /dev/null
+++ b/gremlin-core/src/test/resources/log4j-test.properties
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+log4j.rootLogger=WARN, stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/gremlin-driver/src/test/resources/log4j-silent.properties
----------------------------------------------------------------------
diff --git a/gremlin-driver/src/test/resources/log4j-silent.properties b/gremlin-driver/src/test/resources/log4j-silent.properties
new file mode 100644
index 0000000..9c6148d
--- /dev/null
+++ b/gremlin-driver/src/test/resources/log4j-silent.properties
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+log4j.rootLogger=OFF
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/gremlin-driver/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/gremlin-driver/src/test/resources/log4j-test.properties b/gremlin-driver/src/test/resources/log4j-test.properties
new file mode 100644
index 0000000..ef436fe
--- /dev/null
+++ b/gremlin-driver/src/test/resources/log4j-test.properties
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+log4j.rootLogger=WARN, stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/gremlin-groovy/src/test/resources/log4j-silent.properties
----------------------------------------------------------------------
diff --git a/gremlin-groovy/src/test/resources/log4j-silent.properties b/gremlin-groovy/src/test/resources/log4j-silent.properties
new file mode 100644
index 0000000..9c6148d
--- /dev/null
+++ b/gremlin-groovy/src/test/resources/log4j-silent.properties
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+log4j.rootLogger=OFF
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/gremlin-groovy/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/gremlin-groovy/src/test/resources/log4j-test.properties b/gremlin-groovy/src/test/resources/log4j-test.properties
new file mode 100644
index 0000000..ef436fe
--- /dev/null
+++ b/gremlin-groovy/src/test/resources/log4j-test.properties
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+log4j.rootLogger=WARN, stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinServerAuthIntegrateOldTest.java
----------------------------------------------------------------------
diff --git a/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinServerAuthIntegrateOldTest.java b/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinServerAuthIntegrateOldTest.java
deleted file mode 100644
index 5b09425..0000000
--- a/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinServerAuthIntegrateOldTest.java
+++ /dev/null
@@ -1,235 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.server;
-
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.apache.tinkerpop.gremlin.driver.Client;
-import org.apache.tinkerpop.gremlin.driver.Cluster;
-import org.apache.tinkerpop.gremlin.driver.exception.ResponseException;
-import org.apache.tinkerpop.gremlin.driver.ser.Serializers;
-import org.apache.tinkerpop.gremlin.server.auth.SimpleAuthenticator;
-import org.ietf.jgss.GSSException;
-import org.junit.Test;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeoutException;
-
-import static org.hamcrest.CoreMatchers.startsWith;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-/**
- * @author Stephen Mallette (http://stephen.genoprime.com)
- * @deprecated As of release 3.1.1-incubating, replaced by {@link GremlinServerAuthIntegrateTest}
- * @see <a href="https://issues.apache.org/jira/browse/TINKERPOP3-981">TINKERPOP3-981</a>
- */
-@Deprecated
-public class GremlinServerAuthIntegrateOldTest extends AbstractGremlinServerIntegrationTest {
-
- /**
- * Configure specific Gremlin Server settings for specific tests.
- */
- @Override
- public Settings overrideSettings(final Settings settings) {
- final Settings.AuthenticationSettings authSettings = new Settings.AuthenticationSettings();
- authSettings.className = SimpleAuthenticator.class.getName();
-
- // use a credentials graph with one user in it: stephen/password
- final Map<String,Object> authConfig = new HashMap<>();
- authConfig.put(SimpleAuthenticator.CONFIG_CREDENTIALS_DB, "conf/tinkergraph-empty.properties");
- authConfig.put(SimpleAuthenticator.CONFIG_CREDENTIALS_LOCATION, "data/credentials.kryo");
-
- authSettings.config = authConfig;
- settings.authentication = authSettings;
-
- final String nameOfTest = name.getMethodName();
- switch (nameOfTest) {
- case "shouldAuthenticateOverSslWithPlainText":
- case "shouldFailIfSslEnabledOnServerButNotClient":
- final Settings.SslSettings sslConfig = new Settings.SslSettings();
- sslConfig.enabled = true;
- settings.ssl = sslConfig;
- break;
- }
-
- return settings;
- }
-
- @Test
- public void shouldFailIfSslEnabledOnServerButNotClient() throws Exception {
- final Cluster cluster = Cluster.build().create();
- final Client client = cluster.connect();
-
- try {
- client.submit("1+1").all().get();
- fail("This should not succeed as the client did not enable SSL");
- } catch(Exception ex) {
- final Throwable root = ExceptionUtils.getRootCause(ex);
- assertEquals(TimeoutException.class, root.getClass());
- assertEquals("Timed out waiting for an available host.", root.getMessage());
- } finally {
- cluster.close();
- }
- }
-
- @Test
- public void shouldAuthenticateWithPlainText() throws Exception {
- final Cluster cluster = Cluster.build().credentials("stephen", "password").create();
- final Client client = cluster.connect();
-
- try {
- assertEquals(2, client.submit("1+1").all().get().get(0).getInt());
- assertEquals(3, client.submit("1+2").all().get().get(0).getInt());
- assertEquals(4, client.submit("1+3").all().get().get(0).getInt());
- } finally {
- cluster.close();
- }
- }
-
- @Test
- public void shouldAuthenticateOverSslWithPlainText() throws Exception {
- final Cluster cluster = Cluster.build()
- .enableSsl(true)
- .credentials("stephen", "password").create();
- final Client client = cluster.connect();
-
- try {
- assertEquals(2, client.submit("1+1").all().get().get(0).getInt());
- assertEquals(3, client.submit("1+2").all().get().get(0).getInt());
- assertEquals(4, client.submit("1+3").all().get().get(0).getInt());
- } finally {
- cluster.close();
- }
- }
-
- @Test
- public void shouldFailAuthenticateWithPlainTextNoCredentials() throws Exception {
- final Cluster cluster = Cluster.build().create();
- final Client client = cluster.connect();
-
- try {
- client.submit("1+1").all().get();
- fail("This should not succeed as the client did not provide credentials");
- } catch(Exception ex) {
- final Throwable root = ExceptionUtils.getRootCause(ex);
- assertEquals(GSSException.class, root.getClass());
- assertThat(root.getMessage(), startsWith("Invalid name provided"));
- } finally {
- cluster.close();
- }
- }
-
- @Test
- public void shouldFailAuthenticateWithPlainTextBadPassword() throws Exception {
- final Cluster cluster = Cluster.build().credentials("stephen", "bad").create();
- final Client client = cluster.connect();
-
- try {
- client.submit("1+1").all().get();
- fail("This should not succeed as the client did not provide valid credentials");
- } catch(Exception ex) {
- final Throwable root = ExceptionUtils.getRootCause(ex);
- assertEquals(ResponseException.class, root.getClass());
- assertEquals("Username and/or password are incorrect", root.getMessage());
- } finally {
- cluster.close();
- }
- }
-
- @Test
- public void shouldFailAuthenticateWithPlainTextBadUsername() throws Exception {
- final Cluster cluster = Cluster.build().credentials("marko", "password").create();
- final Client client = cluster.connect();
-
- try {
- client.submit("1+1").all();
- } catch(Exception ex) {
- final Throwable root = ExceptionUtils.getRootCause(ex);
- assertEquals(ResponseException.class, root.getClass());
- assertEquals("Username and/or password are incorrect", root.getMessage());
- } finally {
- cluster.close();
- }
- }
-
- @Test
- public void shouldAuthenticateWithPlainTextOverJSONSerialization() throws Exception {
- final Cluster cluster = Cluster.build().serializer(Serializers.GRAPHSON).credentials("stephen", "password").create();
- final Client client = cluster.connect();
-
- try {
- assertEquals(2, client.submit("1+1").all().get().get(0).getInt());
- assertEquals(3, client.submit("1+2").all().get().get(0).getInt());
- assertEquals(4, client.submit("1+3").all().get().get(0).getInt());
- } finally {
- cluster.close();
- }
- }
-
- @Test
- public void shouldAuthenticateWithPlainTextOverGraphSONSerialization() throws Exception {
- final Cluster cluster = Cluster.build().serializer(Serializers.GRAPHSON_V1D0).credentials("stephen", "password").create();
- final Client client = cluster.connect();
-
- try {
- assertEquals(2, client.submit("1+1").all().get().get(0).getInt());
- assertEquals(3, client.submit("1+2").all().get().get(0).getInt());
- assertEquals(4, client.submit("1+3").all().get().get(0).getInt());
- } finally {
- cluster.close();
- }
- }
-
- @Test
- public void shouldAuthenticateAndWorkWithVariablesOverJsonSerialization() throws Exception {
- final Cluster cluster = Cluster.build().serializer(Serializers.GRAPHSON).credentials("stephen", "password").create();
- final Client client = cluster.connect(name.getMethodName());
-
- try {
- Map vertex = (Map) client.submit("v=graph.addVertex(\"name\", \"stephen\")").all().get().get(0).getObject();
- Map<String, List<Map>> properties = (Map) vertex.get("properties");
- assertEquals("stephen", properties.get("name").get(0).get("value"));
-
- final Map vpName = (Map)client.submit("v.property('name')").all().get().get(0).getObject();
- assertEquals("stephen", vpName.get("value"));
- } finally {
- cluster.close();
- }
- }
-
- @Test
- public void shouldAuthenticateAndWorkWithVariablesOverGraphSONSerialization() throws Exception {
- final Cluster cluster = Cluster.build().serializer(Serializers.GRAPHSON_V1D0).credentials("stephen", "password").create();
- final Client client = cluster.connect(name.getMethodName());
-
- try {
- Map vertex = (Map) client.submit("v=graph.addVertex('name', 'stephen')").all().get().get(0).getObject();
- Map<String, List<Map>> properties = (Map) vertex.get("properties");
- assertEquals("stephen", properties.get("name").get(0).get("value"));
-
- final Map vpName = (Map)client.submit("v.property('name')").all().get().get(0).getObject();
- assertEquals("stephen", vpName.get("value"));
- } finally {
- cluster.close();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinServerAuthOldIntegrateTest.java
----------------------------------------------------------------------
diff --git a/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinServerAuthOldIntegrateTest.java b/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinServerAuthOldIntegrateTest.java
new file mode 100644
index 0000000..cd3a431
--- /dev/null
+++ b/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinServerAuthOldIntegrateTest.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.server;
+
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.tinkerpop.gremlin.driver.Client;
+import org.apache.tinkerpop.gremlin.driver.Cluster;
+import org.apache.tinkerpop.gremlin.driver.exception.ResponseException;
+import org.apache.tinkerpop.gremlin.driver.ser.Serializers;
+import org.apache.tinkerpop.gremlin.server.auth.SimpleAuthenticator;
+import org.ietf.jgss.GSSException;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeoutException;
+
+import static org.hamcrest.CoreMatchers.startsWith;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * @author Stephen Mallette (http://stephen.genoprime.com)
+ * @deprecated As of release 3.1.1-incubating, replaced by {@link GremlinServerAuthIntegrateTest}
+ * @see <a href="https://issues.apache.org/jira/browse/TINKERPOP3-981">TINKERPOP3-981</a>
+ */
+@Deprecated
+public class GremlinServerAuthOldIntegrateTest extends AbstractGremlinServerIntegrationTest {
+
+ /**
+ * Configure specific Gremlin Server settings for specific tests.
+ */
+ @Override
+ public Settings overrideSettings(final Settings settings) {
+ final Settings.AuthenticationSettings authSettings = new Settings.AuthenticationSettings();
+ authSettings.className = SimpleAuthenticator.class.getName();
+
+ // use a credentials graph with one user in it: stephen/password
+ final Map<String,Object> authConfig = new HashMap<>();
+ authConfig.put(SimpleAuthenticator.CONFIG_CREDENTIALS_DB, "conf/tinkergraph-empty.properties");
+ authConfig.put(SimpleAuthenticator.CONFIG_CREDENTIALS_LOCATION, "data/credentials.kryo");
+
+ authSettings.config = authConfig;
+ settings.authentication = authSettings;
+
+ final String nameOfTest = name.getMethodName();
+ switch (nameOfTest) {
+ case "shouldAuthenticateOverSslWithPlainText":
+ case "shouldFailIfSslEnabledOnServerButNotClient":
+ final Settings.SslSettings sslConfig = new Settings.SslSettings();
+ sslConfig.enabled = true;
+ settings.ssl = sslConfig;
+ break;
+ }
+
+ return settings;
+ }
+
+ @Test
+ public void shouldFailIfSslEnabledOnServerButNotClient() throws Exception {
+ final Cluster cluster = Cluster.build().create();
+ final Client client = cluster.connect();
+
+ try {
+ client.submit("1+1").all().get();
+ fail("This should not succeed as the client did not enable SSL");
+ } catch(Exception ex) {
+ final Throwable root = ExceptionUtils.getRootCause(ex);
+ assertEquals(TimeoutException.class, root.getClass());
+ assertEquals("Timed out waiting for an available host.", root.getMessage());
+ } finally {
+ cluster.close();
+ }
+ }
+
+ @Test
+ public void shouldAuthenticateWithPlainText() throws Exception {
+ final Cluster cluster = Cluster.build().credentials("stephen", "password").create();
+ final Client client = cluster.connect();
+
+ try {
+ assertEquals(2, client.submit("1+1").all().get().get(0).getInt());
+ assertEquals(3, client.submit("1+2").all().get().get(0).getInt());
+ assertEquals(4, client.submit("1+3").all().get().get(0).getInt());
+ } finally {
+ cluster.close();
+ }
+ }
+
+ @Test
+ public void shouldAuthenticateOverSslWithPlainText() throws Exception {
+ final Cluster cluster = Cluster.build()
+ .enableSsl(true)
+ .credentials("stephen", "password").create();
+ final Client client = cluster.connect();
+
+ try {
+ assertEquals(2, client.submit("1+1").all().get().get(0).getInt());
+ assertEquals(3, client.submit("1+2").all().get().get(0).getInt());
+ assertEquals(4, client.submit("1+3").all().get().get(0).getInt());
+ } finally {
+ cluster.close();
+ }
+ }
+
+ @Test
+ public void shouldFailAuthenticateWithPlainTextNoCredentials() throws Exception {
+ final Cluster cluster = Cluster.build().create();
+ final Client client = cluster.connect();
+
+ try {
+ client.submit("1+1").all().get();
+ fail("This should not succeed as the client did not provide credentials");
+ } catch(Exception ex) {
+ final Throwable root = ExceptionUtils.getRootCause(ex);
+ assertEquals(GSSException.class, root.getClass());
+ assertThat(root.getMessage(), startsWith("Invalid name provided"));
+ } finally {
+ cluster.close();
+ }
+ }
+
+ @Test
+ public void shouldFailAuthenticateWithPlainTextBadPassword() throws Exception {
+ final Cluster cluster = Cluster.build().credentials("stephen", "bad").create();
+ final Client client = cluster.connect();
+
+ try {
+ client.submit("1+1").all().get();
+ fail("This should not succeed as the client did not provide valid credentials");
+ } catch(Exception ex) {
+ final Throwable root = ExceptionUtils.getRootCause(ex);
+ assertEquals(ResponseException.class, root.getClass());
+ assertEquals("Username and/or password are incorrect", root.getMessage());
+ } finally {
+ cluster.close();
+ }
+ }
+
+ @Test
+ public void shouldFailAuthenticateWithPlainTextBadUsername() throws Exception {
+ final Cluster cluster = Cluster.build().credentials("marko", "password").create();
+ final Client client = cluster.connect();
+
+ try {
+ client.submit("1+1").all();
+ } catch(Exception ex) {
+ final Throwable root = ExceptionUtils.getRootCause(ex);
+ assertEquals(ResponseException.class, root.getClass());
+ assertEquals("Username and/or password are incorrect", root.getMessage());
+ } finally {
+ cluster.close();
+ }
+ }
+
+ @Test
+ public void shouldAuthenticateWithPlainTextOverJSONSerialization() throws Exception {
+ final Cluster cluster = Cluster.build().serializer(Serializers.GRAPHSON).credentials("stephen", "password").create();
+ final Client client = cluster.connect();
+
+ try {
+ assertEquals(2, client.submit("1+1").all().get().get(0).getInt());
+ assertEquals(3, client.submit("1+2").all().get().get(0).getInt());
+ assertEquals(4, client.submit("1+3").all().get().get(0).getInt());
+ } finally {
+ cluster.close();
+ }
+ }
+
+ @Test
+ public void shouldAuthenticateWithPlainTextOverGraphSONSerialization() throws Exception {
+ final Cluster cluster = Cluster.build().serializer(Serializers.GRAPHSON_V1D0).credentials("stephen", "password").create();
+ final Client client = cluster.connect();
+
+ try {
+ assertEquals(2, client.submit("1+1").all().get().get(0).getInt());
+ assertEquals(3, client.submit("1+2").all().get().get(0).getInt());
+ assertEquals(4, client.submit("1+3").all().get().get(0).getInt());
+ } finally {
+ cluster.close();
+ }
+ }
+
+ @Test
+ public void shouldAuthenticateAndWorkWithVariablesOverJsonSerialization() throws Exception {
+ final Cluster cluster = Cluster.build().serializer(Serializers.GRAPHSON).credentials("stephen", "password").create();
+ final Client client = cluster.connect(name.getMethodName());
+
+ try {
+ Map vertex = (Map) client.submit("v=graph.addVertex(\"name\", \"stephen\")").all().get().get(0).getObject();
+ Map<String, List<Map>> properties = (Map) vertex.get("properties");
+ assertEquals("stephen", properties.get("name").get(0).get("value"));
+
+ final Map vpName = (Map)client.submit("v.property('name')").all().get().get(0).getObject();
+ assertEquals("stephen", vpName.get("value"));
+ } finally {
+ cluster.close();
+ }
+ }
+
+ @Test
+ public void shouldAuthenticateAndWorkWithVariablesOverGraphSONSerialization() throws Exception {
+ final Cluster cluster = Cluster.build().serializer(Serializers.GRAPHSON_V1D0).credentials("stephen", "password").create();
+ final Client client = cluster.connect(name.getMethodName());
+
+ try {
+ Map vertex = (Map) client.submit("v=graph.addVertex('name', 'stephen')").all().get().get(0).getObject();
+ Map<String, List<Map>> properties = (Map) vertex.get("properties");
+ assertEquals("stephen", properties.get("name").get(0).get("value"));
+
+ final Map vpName = (Map)client.submit("v.property('name')").all().get().get(0).getObject();
+ assertEquals("stephen", vpName.get("value"));
+ } finally {
+ cluster.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/gremlin-server/src/test/resources/log4j-silent.properties
----------------------------------------------------------------------
diff --git a/gremlin-server/src/test/resources/log4j-silent.properties b/gremlin-server/src/test/resources/log4j-silent.properties
new file mode 100644
index 0000000..9c6148d
--- /dev/null
+++ b/gremlin-server/src/test/resources/log4j-silent.properties
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+log4j.rootLogger=OFF
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/gremlin-server/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/gremlin-server/src/test/resources/log4j-test.properties b/gremlin-server/src/test/resources/log4j-test.properties
new file mode 100644
index 0000000..ef436fe
--- /dev/null
+++ b/gremlin-server/src/test/resources/log4j-test.properties
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+log4j.rootLogger=WARN, stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/gremlin-test/pom.xml
----------------------------------------------------------------------
diff --git a/gremlin-test/pom.xml b/gremlin-test/pom.xml
index 8ae4e40..e9aacad 100644
--- a/gremlin-test/pom.xml
+++ b/gremlin-test/pom.xml
@@ -72,6 +72,12 @@ limitations under the License.
</directory>
</resource>
</resources>
+ <testResources>
+ <testResource>
+ <directory>${basedir}/src/test/resources
+ </directory>
+ </testResource>
+ </testResources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGremlinTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGremlinTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGremlinTest.java
index 97dfea2..ab39572 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGremlinTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGremlinTest.java
@@ -191,6 +191,12 @@ public abstract class AbstractGremlinTest {
graph.tx().commit();
}
+ /**
+ * This method does not appear to be used in TinkerPop core. It was used at one time by Neo4j tests, but wasn't
+ * really a great pattern and was eventually removed.
+ *
+ * @deprecated as of 3.1.1-incubating, and is not replaced
+ */
public void tryRandomCommit(final Graph graph) {
if (graph.features().graph().supportsTransactions() && new Random().nextBoolean())
graph.tx().commit();
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/gremlin-test/src/test/resources/log4j-silent.properties
----------------------------------------------------------------------
diff --git a/gremlin-test/src/test/resources/log4j-silent.properties b/gremlin-test/src/test/resources/log4j-silent.properties
new file mode 100644
index 0000000..9d4f1d4
--- /dev/null
+++ b/gremlin-test/src/test/resources/log4j-silent.properties
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+log4j.rootLogger=OFF
+
+# need to turn this on as this test actually tests "logging" - needs INFO at minimum for tests to pass
+log4j.logger.org.apache.tinkerpop.gremlin.util.Log4jRecordingAppenderTest=INFO
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/gremlin-test/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/gremlin-test/src/test/resources/log4j-test.properties b/gremlin-test/src/test/resources/log4j-test.properties
new file mode 100644
index 0000000..3d02711
--- /dev/null
+++ b/gremlin-test/src/test/resources/log4j-test.properties
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+log4j.rootLogger=WARN, stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n
+
+# need to turn this on as this test actually tests "logging" - needs INFO at minimum for tests to pass
+log4j.logger.org.apache.tinkerpop.gremlin.util.Log4jRecordingAppenderTest=INFO
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/hadoop-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/pom.xml b/hadoop-gremlin/pom.xml
index b2f7d69..3542275 100644
--- a/hadoop-gremlin/pom.xml
+++ b/hadoop-gremlin/pom.xml
@@ -198,6 +198,10 @@ limitations under the License.
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
</plugin>
<plugin>
@@ -222,20 +226,6 @@ limitations under the License.
<invokeDynamic>true</invokeDynamic>
</configuration>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <!-- log4j.configuration>log4j-core.properties</log4j.configuration> -->
- <!--<argLine>-Xmx2048M</argLine>-->
- <excludes>
- <exclude>**/*IntegrateTest.java</exclude>
- <exclude>**/*PerformanceTest.java</exclude>
- <!-- this is technically a member of the integration test suite -->
- <exclude>**/HadoopGremlinPluginTest.java</exclude>
- </excludes>
- </configuration>
- </plugin>
</plugins>
</build>
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPluginCheck.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPluginCheck.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPluginCheck.java
new file mode 100644
index 0000000..4108e35
--- /dev/null
+++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPluginCheck.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.hadoop.groovy.plugin;
+
+import org.apache.tinkerpop.gremlin.AbstractGremlinTest;
+import org.apache.tinkerpop.gremlin.LoadGraphWith;
+import org.apache.tinkerpop.gremlin.groovy.plugin.RemoteAcceptor;
+import org.apache.tinkerpop.gremlin.groovy.util.TestableConsolePluginAcceptor;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.HadoopGraphProvider;
+import org.apache.tinkerpop.gremlin.process.AbstractGremlinProcessTest;
+import org.apache.tinkerpop.gremlin.process.computer.KeyValue;
+import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.step.sideEffect.mapreduce.TraverserMapReduce;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.UUID;
+
+import static org.junit.Assert.*;
+
+/**
+ * This is an test that is mean to be used in the context of the {@link HadoopPluginSuite} and shouldn't be
+ * executed on its own.
+ *
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public class HadoopGremlinPluginCheck extends AbstractGremlinTest {
+
+ @Before
+ public void setupTest() {
+ try {
+ this.console = new TestableConsolePluginAcceptor();
+ final HadoopGremlinPlugin plugin = new HadoopGremlinPlugin();
+ plugin.pluginTo(this.console);
+ this.remote = (HadoopRemoteAcceptor) plugin.remoteAcceptor().get();
+ } catch (final Exception e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ }
+
+ ///////////////////
+
+ private HadoopRemoteAcceptor remote;
+ private TestableConsolePluginAcceptor console;
+
+ @Test
+ @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
+ public void shouldSupportRemoteTraversal() throws Exception {
+ this.console.addBinding("graph", this.graph);
+ this.console.addBinding("g", this.g);
+ this.remote.connect(Arrays.asList("graph", "g"));
+ //
+ Traversal<?, ?> traversal = (Traversal<?, ?>) this.remote.submit(Arrays.asList("g.V().count()"));
+ assertEquals(6L, traversal.next());
+ assertFalse(traversal.hasNext());
+ assertNotNull(this.console.getBindings().get(RemoteAcceptor.RESULT));
+ }
+
+ @Test
+ @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
+ public void shouldSupportRemoteSugarTraversal() throws Exception {
+ this.console.addBinding("graph", this.graph);
+ this.console.addBinding("g", this.g);
+ this.remote.connect(Arrays.asList("graph"));
+ //
+ this.remote.configure(Arrays.asList("useSugar", "true"));
+ this.remote.connect(Arrays.asList("graph", "g"));
+ Traversal<?, ?> traversal = (Traversal<?, ?>) this.remote.submit(Arrays.asList("g.V.name.map{it.length()}.sum"));
+ assertEquals(28l, traversal.next());
+ assertFalse(traversal.hasNext());
+ assertNotNull(this.console.getBindings().get(RemoteAcceptor.RESULT));
+ }
+
+ @Test
+ @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
+ public void shouldSupportHDFSMethods() throws Exception {
+ ////////////////
+ List<String> ls = (List<String>) this.console.eval("hdfs.ls()");
+ for (final String line : ls) {
+ assertTrue(line.startsWith("-") || line.startsWith("r") || line.startsWith("w") || line.startsWith("x"));
+ assertEquals(" ", line.substring(9, 10));
+ }
+ ls = (List<String>) this.console.eval("local.ls()");
+ for (final String line : ls) {
+ assertTrue(line.startsWith("-") || line.startsWith("r") || line.startsWith("w") || line.startsWith("x"));
+ assertEquals(" ", line.substring(9, 10));
+ }
+ ////////////////
+ this.console.eval("hdfs.copyFromLocal('" + HadoopGraphProvider.PATHS.get("tinkerpop-classic.txt") + "', 'target/tinkerpop-classic.txt')");
+ assertTrue((Boolean) this.console.eval("hdfs.exists('target/tinkerpop-classic.txt')"));
+ ////////////////
+ List<String> head = IteratorUtils.asList(this.console.eval("hdfs.head('target/tinkerpop-classic.txt')"));
+ assertEquals(6, head.size());
+ for (final String line : head) {
+ assertEquals(":", line.substring(1, 2));
+ assertTrue(Integer.valueOf(line.substring(0, 1)) <= 6);
+ }
+ head = IteratorUtils.asList(this.console.eval("hdfs.head('target/tinkerpop-classic.txt',3)"));
+ assertEquals(3, head.size());
+ for (final String line : head) {
+ assertEquals(":", line.substring(1, 2));
+ assertTrue(Integer.valueOf(line.substring(0, 1)) <= 3);
+ }
+ ////////////////
+ this.console.eval("hdfs.rm('target/tinkerpop-classic.txt')");
+ assertFalse((Boolean) this.console.eval("hdfs.exists('target/tinkerpop-classic.txt')"));
+ ////////////////
+ this.console.addBinding("graph", this.graph);
+ this.console.addBinding("g", this.g);
+ this.remote.connect(Arrays.asList("graph", "g"));
+ Traversal<Vertex, String> traversal = (Traversal<Vertex, String>) this.remote.submit(Arrays.asList("g.V().hasLabel('person').group('m').by('age').by('name').out('knows').out('created').values('name')"));
+ AbstractGremlinProcessTest.checkResults(Arrays.asList("ripple", "lop"), traversal);
+ assertTrue((Boolean) this.console.eval("hdfs.exists('target/test-output/m')"));
+ assertTrue((Boolean) this.console.eval("hdfs.exists('target/test-output/" + TraverserMapReduce.TRAVERSERS + "')"));
+ final List<KeyValue<Integer, Collection<String>>> mList = IteratorUtils.asList(this.console.eval("hdfs.head('target/test-output/m',ObjectWritable)"));
+ assertEquals(4, mList.size());
+ mList.forEach(keyValue -> {
+ if (keyValue.getKey().equals(29))
+ assertTrue(keyValue.getValue().contains("marko"));
+ else if (keyValue.getKey().equals(35))
+ assertTrue(keyValue.getValue().contains("peter"));
+ else if (keyValue.getKey().equals(32))
+ assertTrue(keyValue.getValue().contains("josh"));
+ else if (keyValue.getKey().equals(27))
+ assertTrue(keyValue.getValue().contains("vadas"));
+ else
+ throw new IllegalStateException("The provided key/value is unknown: " + keyValue);
+ });
+ final List<KeyValue<MapReduce.NullObject, Traverser<String>>> traversersList = IteratorUtils.asList(this.console.eval("hdfs.head('target/test-output/" + TraverserMapReduce.TRAVERSERS + "',ObjectWritable)"));
+ assertEquals(2, traversersList.size());
+ traversersList.forEach(keyValue -> {
+ assertEquals(MapReduce.NullObject.instance(), keyValue.getKey());
+ final String name = keyValue.getValue().get();
+ assertTrue(name.equals("ripple") || name.equals("lop"));
+ });
+ ////////////////
+ }
+
+ @Test
+ @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
+ public void shouldGracefullyHandleBadGremlinHadoopLibs() throws Exception {
+ System.setProperty(Constants.HADOOP_GREMLIN_LIBS, "test/" + UUID.randomUUID());
+ this.graph.configuration().setProperty(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, true);
+ this.console.addBinding("graph", this.graph);
+ this.console.addBinding("g", this.g);
+ this.remote.connect(Arrays.asList("graph", "g"));
+ Traversal<?, ?> traversal = (Traversal<?, ?>) this.remote.submit(Arrays.asList("g.V()"));
+ assertEquals(6, IteratorUtils.count(traversal));
+ assertNotNull(this.console.getBindings().get(RemoteAcceptor.RESULT));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPluginTest.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPluginTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPluginTest.java
deleted file mode 100644
index 0d7da18..0000000
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPluginTest.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.hadoop.groovy.plugin;
-
-import org.apache.tinkerpop.gremlin.AbstractGremlinTest;
-import org.apache.tinkerpop.gremlin.LoadGraphWith;
-import org.apache.tinkerpop.gremlin.groovy.plugin.RemoteAcceptor;
-import org.apache.tinkerpop.gremlin.groovy.util.TestableConsolePluginAcceptor;
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.HadoopGraphProvider;
-import org.apache.tinkerpop.gremlin.process.AbstractGremlinProcessTest;
-import org.apache.tinkerpop.gremlin.process.computer.KeyValue;
-import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
-import org.apache.tinkerpop.gremlin.process.computer.traversal.step.sideEffect.mapreduce.TraverserMapReduce;
-import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
-import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-import java.util.UUID;
-
-import static org.junit.Assert.*;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public class HadoopGremlinPluginTest extends AbstractGremlinTest {
-
- @Before
- public void setupTest() {
- try {
- this.console = new TestableConsolePluginAcceptor();
- final HadoopGremlinPlugin plugin = new HadoopGremlinPlugin();
- plugin.pluginTo(this.console);
- this.remote = (HadoopRemoteAcceptor) plugin.remoteAcceptor().get();
- } catch (final Exception e) {
- throw new IllegalStateException(e.getMessage(), e);
- }
- }
-
- ///////////////////
-
- private HadoopRemoteAcceptor remote;
- private TestableConsolePluginAcceptor console;
-
- @Test
- @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
- public void shouldSupportRemoteTraversal() throws Exception {
- this.console.addBinding("graph", this.graph);
- this.console.addBinding("g", this.g);
- this.remote.connect(Arrays.asList("graph", "g"));
- //
- Traversal<?, ?> traversal = (Traversal<?, ?>) this.remote.submit(Arrays.asList("g.V().count()"));
- assertEquals(6L, traversal.next());
- assertFalse(traversal.hasNext());
- assertNotNull(this.console.getBindings().get(RemoteAcceptor.RESULT));
- }
-
- @Test
- @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
- public void shouldSupportRemoteSugarTraversal() throws Exception {
- this.console.addBinding("graph", this.graph);
- this.console.addBinding("g", this.g);
- this.remote.connect(Arrays.asList("graph"));
- //
- this.remote.configure(Arrays.asList("useSugar", "true"));
- this.remote.connect(Arrays.asList("graph", "g"));
- Traversal<?, ?> traversal = (Traversal<?, ?>) this.remote.submit(Arrays.asList("g.V.name.map{it.length()}.sum"));
- assertEquals(28l, traversal.next());
- assertFalse(traversal.hasNext());
- assertNotNull(this.console.getBindings().get(RemoteAcceptor.RESULT));
- }
-
- @Test
- @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
- public void shouldSupportHDFSMethods() throws Exception {
- ////////////////
- List<String> ls = (List<String>) this.console.eval("hdfs.ls()");
- for (final String line : ls) {
- assertTrue(line.startsWith("-") || line.startsWith("r") || line.startsWith("w") || line.startsWith("x"));
- assertEquals(" ", line.substring(9, 10));
- }
- ls = (List<String>) this.console.eval("local.ls()");
- for (final String line : ls) {
- assertTrue(line.startsWith("-") || line.startsWith("r") || line.startsWith("w") || line.startsWith("x"));
- assertEquals(" ", line.substring(9, 10));
- }
- ////////////////
- this.console.eval("hdfs.copyFromLocal('" + HadoopGraphProvider.PATHS.get("tinkerpop-classic.txt") + "', 'target/tinkerpop-classic.txt')");
- assertTrue((Boolean) this.console.eval("hdfs.exists('target/tinkerpop-classic.txt')"));
- ////////////////
- List<String> head = IteratorUtils.asList(this.console.eval("hdfs.head('target/tinkerpop-classic.txt')"));
- assertEquals(6, head.size());
- for (final String line : head) {
- assertEquals(":", line.substring(1, 2));
- assertTrue(Integer.valueOf(line.substring(0, 1)) <= 6);
- }
- head = IteratorUtils.asList(this.console.eval("hdfs.head('target/tinkerpop-classic.txt',3)"));
- assertEquals(3, head.size());
- for (final String line : head) {
- assertEquals(":", line.substring(1, 2));
- assertTrue(Integer.valueOf(line.substring(0, 1)) <= 3);
- }
- ////////////////
- this.console.eval("hdfs.rm('target/tinkerpop-classic.txt')");
- assertFalse((Boolean) this.console.eval("hdfs.exists('target/tinkerpop-classic.txt')"));
- ////////////////
- this.console.addBinding("graph", this.graph);
- this.console.addBinding("g", this.g);
- this.remote.connect(Arrays.asList("graph", "g"));
- Traversal<Vertex, String> traversal = (Traversal<Vertex, String>) this.remote.submit(Arrays.asList("g.V().hasLabel('person').group('m').by('age').by('name').out('knows').out('created').values('name')"));
- AbstractGremlinProcessTest.checkResults(Arrays.asList("ripple", "lop"), traversal);
- assertTrue((Boolean) this.console.eval("hdfs.exists('target/test-output/m')"));
- assertTrue((Boolean) this.console.eval("hdfs.exists('target/test-output/" + TraverserMapReduce.TRAVERSERS + "')"));
- final List<KeyValue<Integer, Collection<String>>> mList = IteratorUtils.asList(this.console.eval("hdfs.head('target/test-output/m',ObjectWritable)"));
- assertEquals(4, mList.size());
- mList.forEach(keyValue -> {
- if (keyValue.getKey().equals(29))
- assertTrue(keyValue.getValue().contains("marko"));
- else if (keyValue.getKey().equals(35))
- assertTrue(keyValue.getValue().contains("peter"));
- else if (keyValue.getKey().equals(32))
- assertTrue(keyValue.getValue().contains("josh"));
- else if (keyValue.getKey().equals(27))
- assertTrue(keyValue.getValue().contains("vadas"));
- else
- throw new IllegalStateException("The provided key/value is unknown: " + keyValue);
- });
- final List<KeyValue<MapReduce.NullObject, Traverser<String>>> traversersList = IteratorUtils.asList(this.console.eval("hdfs.head('target/test-output/" + TraverserMapReduce.TRAVERSERS + "',ObjectWritable)"));
- assertEquals(2, traversersList.size());
- traversersList.forEach(keyValue -> {
- assertEquals(MapReduce.NullObject.instance(), keyValue.getKey());
- final String name = keyValue.getValue().get();
- assertTrue(name.equals("ripple") || name.equals("lop"));
- });
- ////////////////
- }
-
- @Test
- @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
- public void shouldGracefullyHandleBadGremlinHadoopLibs() throws Exception {
- System.setProperty(Constants.HADOOP_GREMLIN_LIBS, "test/" + UUID.randomUUID());
- this.graph.configuration().setProperty(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, true);
- this.console.addBinding("graph", this.graph);
- this.console.addBinding("g", this.g);
- this.remote.connect(Arrays.asList("graph", "g"));
- Traversal<?, ?> traversal = (Traversal<?, ?>) this.remote.submit(Arrays.asList("g.V()"));
- assertEquals(6, IteratorUtils.count(traversal));
- assertNotNull(this.console.getBindings().get(RemoteAcceptor.RESULT));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopPluginSuite.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopPluginSuite.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopPluginSuite.java
index 59ba23c..6c4cc20 100644
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopPluginSuite.java
+++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopPluginSuite.java
@@ -29,6 +29,6 @@ import org.junit.runners.model.RunnerBuilder;
*/
public class HadoopPluginSuite extends AbstractGremlinSuite {
public HadoopPluginSuite(final Class<?> klass, final RunnerBuilder builder) throws InitializationError {
- super(klass, builder, new Class<?>[]{HadoopGremlinPluginTest.class}, new Class<?>[]{HadoopGremlinPluginTest.class}, true, TraversalEngine.Type.COMPUTER);
+ super(klass, builder, new Class<?>[]{HadoopGremlinPluginCheck.class}, new Class<?>[]{HadoopGremlinPluginCheck.class}, true, TraversalEngine.Type.COMPUTER);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/hadoop-gremlin/src/test/resources/log4j-silent.properties
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/resources/log4j-silent.properties b/hadoop-gremlin/src/test/resources/log4j-silent.properties
new file mode 100644
index 0000000..9c6148d
--- /dev/null
+++ b/hadoop-gremlin/src/test/resources/log4j-silent.properties
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+log4j.rootLogger=OFF
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/hadoop-gremlin/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/resources/log4j-test.properties b/hadoop-gremlin/src/test/resources/log4j-test.properties
new file mode 100644
index 0000000..4b5af31
--- /dev/null
+++ b/hadoop-gremlin/src/test/resources/log4j-test.properties
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+log4j.rootLogger=WARN, stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/neo4j-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/neo4j-gremlin/pom.xml b/neo4j-gremlin/pom.xml
index b8d6289..b13c556 100644
--- a/neo4j-gremlin/pom.xml
+++ b/neo4j-gremlin/pom.xml
@@ -64,6 +64,12 @@ limitations under the License.
</directory>
</resource>
</resources>
+ <testResources>
+ <testResource>
+ <directory>${basedir}/src/test/resources
+ </directory>
+ </testResource>
+ </testResources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/neo4j-gremlin/src/test/resources/log4j-silent.properties
----------------------------------------------------------------------
diff --git a/neo4j-gremlin/src/test/resources/log4j-silent.properties b/neo4j-gremlin/src/test/resources/log4j-silent.properties
new file mode 100644
index 0000000..9c6148d
--- /dev/null
+++ b/neo4j-gremlin/src/test/resources/log4j-silent.properties
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+log4j.rootLogger=OFF
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/neo4j-gremlin/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/neo4j-gremlin/src/test/resources/log4j-test.properties b/neo4j-gremlin/src/test/resources/log4j-test.properties
new file mode 100644
index 0000000..ef436fe
--- /dev/null
+++ b/neo4j-gremlin/src/test/resources/log4j-test.properties
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+log4j.rootLogger=WARN, stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2b2bf960/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0332cdc..371a108 100644
--- a/pom.xml
+++ b/pom.xml
@@ -127,13 +127,6 @@ limitations under the License.
<properties>
<groovy.version>2.4.1</groovy.version>
<junit.version>4.12</junit.version>
- <!-- Directory containing the file ${log4j.properties} defined
- above. Maven interpolates ${project.basedir} in the
- context of child modules, so path this relative to one of
- the modules under the root directory, not the root
- directory itself. -->
- <log4j.properties.dir>${project.basedir}/../gremlin-server/conf/</log4j.properties.dir>
- <log4j.properties>log4j.properties</log4j.properties>
<metrics.version>3.0.2</metrics.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
@@ -144,6 +137,15 @@ limitations under the License.
<commons.configuration.version>1.10</commons.configuration.version>
<java.tuples.version>1.2</java.tuples.version>
<javadoc-plugin.version>2.10.1</javadoc-plugin.version>
+
+ <!--
+ This file will come from the root of each modules test/resources/ directories. log4j-silent.properties will
+ turn off all logging possible while still allowing tests to pass. log4j-test.properties will be more verbose
+ in its output.
+ -->
+ <log4j-test.properties>file:target/test-classes/log4j-test.properties</log4j-test.properties>
+ <log4j-silent.properties>file:target/test-classes/log4j-silent.properties</log4j-silent.properties>
+
<muteTestLogs>false</muteTestLogs>
</properties>
<build>
@@ -316,12 +318,8 @@ limitations under the License.
<artifactId>maven-surefire-plugin</artifactId>
<version>2.17</version>
<configuration>
- <argLine>-Dlog4j.configuration=${log4j.properties} -Dbuild.dir=${project.build.directory}
+ <argLine>-Dlog4j.configuration=${log4j-test.properties} -Dbuild.dir=${project.build.directory}
</argLine>
- <additionalClasspathElements>
- <!-- So that Log4j can find the Log4j properties file -->
- <additionalClasspathElement>${log4j.properties.dir}</additionalClasspathElement>
- </additionalClasspathElements>
<excludes>
<exclude>**/*IntegrateTest.java</exclude>
<exclude>**/*PerformanceTest.java</exclude>
@@ -343,15 +341,11 @@ limitations under the License.
<include>**/*IntegrateTest.java</include>
</includes>
<skipTests>${skipIntegrationTests}</skipTests>
- <argLine>-Dlog4j.configuration=${log4j.properties} -Dhost=localhost -Dport=8182
+ <argLine>-Dlog4j.configuration=${log4j-test.properties} -Dhost=localhost -Dport=8182
-Djub.consumers=CONSOLE,H2 -Djub.db.file=target/performance/h2/benchmarks
-Djub.charts.dir=target/performance/charts -Djub.customkey=${project.parent.version}
-Dbuild.dir=${project.build.directory}
</argLine>
- <additionalClasspathElements>
- <!-- So that Log4j can find the Log4j properties file -->
- <additionalClasspathElement>${log4j.properties.dir}</additionalClasspathElement>
- </additionalClasspathElements>
<summaryFile>target/failsafe-reports/failsafe-integration.xml</summaryFile>
</configuration>
</execution>
@@ -365,15 +359,11 @@ limitations under the License.
<include>**/*PerformanceTest.java</include>
</includes>
<skipTests>${skipPerformanceTests}</skipTests>
- <argLine>-Dlog4j.configuration=${log4j.properties} -Dhost=localhost -Dport=8182
+ <argLine>-Dlog4j.configuration=${log4j-test.properties} -Dhost=localhost -Dport=8182
-Djub.consumers=CONSOLE,H2 -Djub.db.file=target/performance/h2/benchmarks
-Djub.charts.dir=target/performance/charts -Djub.customkey=${project.parent.version}
-Dbuild.dir=${project.build.directory}
</argLine>
- <additionalClasspathElements>
- <!-- So that Log4j can find the Log4j properties file -->
- <additionalClasspathElement>${log4j.properties.dir}</additionalClasspathElement>
- </additionalClasspathElements>
<summaryFile>target/failsafe-reports/failsafe-performance.xml</summaryFile>
</configuration>
</execution>
@@ -418,6 +408,7 @@ limitations under the License.
<version>1.0</version>
</plugin>
<plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>${javadoc-plugin.version}</version>
</plugin>
@@ -914,7 +905,7 @@ limitations under the License.
</profile>
<profile>
- <!-- Continuous Integration - mutes test logs -->
+ <!-- Continuous Integration - mutes test logs and skip javadoc generation to lighten output -->
<id>ci</id>
<activation>
<activeByDefault>false</activeByDefault>
@@ -927,12 +918,22 @@ limitations under the License.
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <version>${javadoc-plugin.version}</version>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.17</version>
<configuration>
<systemPropertyVariables>
<muteTestLogs>true</muteTestLogs>
</systemPropertyVariables>
+ <argLine>-Dlog4j.configuration=${log4j-silent.properties} -Dbuild.dir=${project.build.directory}
+ </argLine>
<excludes>
<exclude>**/*IntegrateTest.java</exclude>
<exclude>**/*PerformanceTest.java</exclude>
[03/31] incubator-tinkerpop git commit: Merge branch 'TINKERPOP3-1013'
Posted by sp...@apache.org.
Merge branch 'TINKERPOP3-1013'
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/6041de66
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/6041de66
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/6041de66
Branch: refs/heads/TINKERPOP3-860
Commit: 6041de6630e4c253b817bd859845812517c89b90
Parents: 59eaa06 e6a1141
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed Dec 2 15:59:13 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed Dec 2 15:59:13 2015 -0700
----------------------------------------------------------------------
.../gremlin/process/traversal/Traverser.java | 9 ++++
.../process/traversal/step/map/MatchStep.java | 56 +++++++++++++-------
.../traversal/step/util/ComputerAwareStep.java | 2 +-
.../traversal/traverser/O_Traverser.java | 37 +++++++++++++
.../traverser/util/EmptyTraverser.java | 6 +++
.../traversal/step/map/MatchStepTest.java | 15 ++++--
6 files changed, 99 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
[25/31] incubator-tinkerpop git commit: Merge remote-tracking branch
'origin/master'
Posted by sp...@apache.org.
Merge remote-tracking branch 'origin/master'
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/641a8cb0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/641a8cb0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/641a8cb0
Branch: refs/heads/TINKERPOP3-860
Commit: 641a8cb045615a173758c096225364cdcb10bc7c
Parents: 00a2965 1134ad2
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Thu Dec 3 15:42:15 2015 -0500
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Dec 3 15:42:15 2015 -0500
----------------------------------------------------------------------
.travis.yml | 3 +-
CHANGELOG.asciidoc | 6 +
docs/src/reference/the-traversal.asciidoc | 33 +++--
.../gremlin/process/traversal/Order.java | 28 +++-
.../traversal/dsl/graph/GraphTraversal.java | 20 ++-
.../traversal/step/map/SelectColumnStep.java | 68 ----------
.../traversal/step/util/FunctionComparator.java | 56 ++++++++
.../tinkerpop/gremlin/structure/Column.java | 39 +++++-
.../step/branch/GroovyLocalTest.groovy | 2 +-
.../traversal/step/map/GroovyOrderTest.groovy | 8 +-
.../process/GroovyProcessComputerSuite.java | 2 +
.../process/GroovyProcessStandardSuite.java | 2 +
.../BulkLoaderVertexProgramTest.java | 2 +-
.../process/traversal/step/map/OrderTest.java | 11 +-
.../structure/hdfs/HadoopEdgeIterator.java | 4 -
.../structure/hdfs/HadoopElementIterator.java | 35 ++---
.../structure/hdfs/HadoopVertexIterator.java | 4 -
.../HadoopGraphStructureStandardTest.java | 35 +++++
.../io/HadoopGraphStructureStandardTest.java | 35 -----
pom.xml | 10 +-
.../process/computer/SparkContextHelper.java | 39 ++++++
.../process/computer/SparkGraphComputer.java | 15 +-
.../computer/payload/MessagePayload.java | 5 +-
.../computer/payload/ViewIncomingPayload.java | 6 +-
.../computer/payload/ViewOutgoingPayload.java | 12 +-
.../spark/structure/io/InputRDDFormat.java | 136 +++++++++++++++++++
.../spark/structure/io/gryo/GryoSerializer.java | 6 +
.../io/gryo/WrappedArraySerializer.java | 46 +++++++
.../gremlin/spark/AbstractSparkTest.java | 43 ++++++
.../process/computer/LocalPropertyTest.java | 8 +-
.../computer/SparkHadoopGraphProvider.java | 22 ++-
.../spark/structure/io/InputOutputRDDTest.java | 7 +-
.../spark/structure/io/InputRDDTest.java | 28 +++-
.../spark/structure/io/OutputRDDTest.java | 7 +-
.../io/PersistedInputOutputRDDTest.java | 14 +-
.../spark/structure/io/ToyGraphInputRDD.java | 73 ++++++++++
.../structure/TinkerGraphPlayTest.java | 7 +-
37 files changed, 663 insertions(+), 214 deletions(-)
----------------------------------------------------------------------
[22/31] incubator-tinkerpop git commit: Updated CHANGELOG.
Posted by sp...@apache.org.
Updated CHANGELOG.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/1134ad2a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/1134ad2a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/1134ad2a
Branch: refs/heads/TINKERPOP3-860
Commit: 1134ad2a50ae4cce7fbcd2b8f362ff762139c7e4
Parents: b2fee75
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Thu Dec 3 12:01:03 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Dec 3 12:48:58 2015 -0700
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1134ad2a/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 869b494..717aa71 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,6 +26,7 @@ image::https://raw.githubusercontent.com/apache/incubator-tinkerpop/master/docs/
TinkerPop 3.1.1 (NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* `Column` now implements `Function` and now `by(valueDecr)` can be replaced with `by(values,decr)` and thus, projection and order are now separated.
* Added `InputRDDFormat` which wraps an `InputRDD` to make it accessible to Hadoop and not just Spark.
* Added `AbstractSparkTest` which handles closing `SparkContext` instances between tests now that we support persisted contexts.
* Fixed a serialization bug in `GryoSerializer` that made it difficult for graph providers to yield `InputRDDs` for `SparkGraphComputer`.
[19/31] incubator-tinkerpop git commit: added more docs to Order Step
around Columm.keys and Column.values. Added a GraphTraversal.by() safeguard
to ensure no ambiguous method call issues.
Posted by sp...@apache.org.
added more docs to Order Step around Columm.keys and Column.values. Added a GraphTraversal.by() safeguard to ensure no ambiguous method call issues.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/4795042e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/4795042e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/4795042e
Branch: refs/heads/TINKERPOP3-860
Commit: 4795042eab7f047cbde57237b9900c43054e0c31
Parents: 447d705
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Thu Dec 3 09:42:27 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Dec 3 12:47:53 2015 -0700
----------------------------------------------------------------------
docs/src/reference/the-traversal.asciidoc | 17 +++++++++++++-
.../gremlin/process/traversal/Order.java | 24 ++++++++++----------
.../traversal/dsl/graph/GraphTraversal.java | 8 ++++---
.../process/traversal/step/map/OrderTest.java | 2 +-
4 files changed, 34 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4795042e/docs/src/reference/the-traversal.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/the-traversal.asciidoc b/docs/src/reference/the-traversal.asciidoc
index 6ce4541..271438b 100644
--- a/docs/src/reference/the-traversal.asciidoc
+++ b/docs/src/reference/the-traversal.asciidoc
@@ -1197,9 +1197,24 @@ g.V().hasLabel('person').order().by(shuffle)
g.V().hasLabel('person').order().by(shuffle)
----
-IMPORTANT: `order(local)` orders the current, local object (not the objects in the traversal stream). This works for
+It is possible to use `order(local)` to order the current local object and not the entire traversal stream. This works for
`Collection`- and `Map`-type objects. For any other object, the object is returned unchanged.
+[gremlin-groovy,modern]
+----
+g.V().values('age').fold().order(local).by(decr) <1>
+g.V().values('age').order(local).by(decr) <2>
+g.V().groupCount().by(inE().count()).order(local).by(values, decr) <3>
+g.V().groupCount().by(inE().count()).order(local).by(keys, incr) <4>
+----
+
+<1> The ages are gathered into a list and then that list is sorted in decreasing order.
+<2> The ages are not gathered and thus `order(local)` is "ordering" single integers and thus, does nothing.
+<3> The `groupCount()` map is ordered by its values in decreasing order.
+<4> The `groupCount()` map is ordered by its keys in increasing order.
+
+NOTE: The `values` and `keys` enums are from `Column` which is used to select "columns" from a `Map`, `Map.Entry`, or `Path`.
+
[[path-step]]
Path Step
~~~~~~~~~
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4795042e/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/Order.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/Order.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/Order.java
index 4e3801a..72fe715 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/Order.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/Order.java
@@ -48,10 +48,10 @@ public enum Order implements Comparator<Object> {
return incr;
}
},
- /**
+ /*
* @deprecated Use {@link org.apache.tinkerpop.gremlin.structure.Column#keys};
- */
- @Deprecated
+ @Deprecated
+ */
keyIncr {
@Override
public int compare(final Object first, final Object second) {
@@ -63,10 +63,10 @@ public enum Order implements Comparator<Object> {
return keyDecr;
}
},
- /**
+ /*
* @deprecated Use {@link org.apache.tinkerpop.gremlin.structure.Column#values};
- */
- @Deprecated
+ *
+ @Deprecated */
valueIncr {
@Override
public int compare(final Object first, final Object second) {
@@ -78,10 +78,10 @@ public enum Order implements Comparator<Object> {
return valueDecr;
}
},
- /**
+ /*
* @deprecated Use {@link org.apache.tinkerpop.gremlin.structure.Column#keys};
- */
- @Deprecated
+ *
+ @Deprecated */
keyDecr {
@Override
public int compare(final Object first, final Object second) {
@@ -93,10 +93,10 @@ public enum Order implements Comparator<Object> {
return keyIncr;
}
},
- /**
+ /*
* @deprecated Use {@link org.apache.tinkerpop.gremlin.structure.Column#values};
- */
- @Deprecated
+ *
+ @Deprecated */
valueDecr {
@Override
public int compare(final Object first, final Object second) {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4795042e/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java
index 3673bce..5dd963c 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java
@@ -1164,7 +1164,7 @@ public interface GraphTraversal<S, E> extends Traversal<S, E> {
}
- ////
+ //// PROJECTION BY-MODULATORS
public default GraphTraversal<S, E> by(final Traversal<?, ?> byTraversal) {
((TraversalParent) this.asAdmin().getEndStep()).addLocalChild(byTraversal.asAdmin());
@@ -1187,7 +1187,7 @@ public interface GraphTraversal<S, E> extends Traversal<S, E> {
return this.by(new ElementValueTraversal<>(elementPropertyKey));
}
- ////
+ //// COMPARATOR BY-MODULATORS
public default GraphTraversal<S, E> by(final Comparator<E> comparator) {
((ComparatorHolder<E>) this.asAdmin().getEndStep()).addComparator(comparator);
@@ -1204,7 +1204,9 @@ public interface GraphTraversal<S, E> extends Traversal<S, E> {
public default <V> GraphTraversal<S, E> by(final Function<Element, V> elementFunctionProjection,
final Comparator<V> elementFunctionValueComparator) {
- return this.by((Comparator) new ElementFunctionComparator<>(elementFunctionProjection, elementFunctionValueComparator));
+ return ((Function) elementFunctionProjection) instanceof Column ?
+ this.by((Column) ((Function) elementFunctionProjection), elementFunctionValueComparator) :
+ this.by((Comparator) new ElementFunctionComparator<>(elementFunctionProjection, elementFunctionValueComparator));
}
public default <V> GraphTraversal<S, E> by(final String elementPropertyProjection,
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4795042e/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderTest.java
index 4475b7c..b69c625 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/OrderTest.java
@@ -364,7 +364,7 @@ public abstract class OrderTest extends AbstractGremlinProcessTest {
map.put(3, (int) v.get().value("age") * 3);
map.put(4, (int) v.get().value("age"));
return map;
- }).order(Scope.local).by(Column.values, Order.decr).by(Column.keys,Order.incr);
+ }).order(Scope.local).by(Column.values, Order.decr).by(Column.keys, Order.incr);
}
@Override