You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by dk...@apache.org on 2015/12/18 19:04:11 UTC
[2/8] incubator-tinkerpop git commit: Added another `BulkLoader`
implementation (`OneTimeBulkLoader`) to verify that `id.toString()` vertex
lookups still work properly if no temporary property is written to the target
graph.
Added another `BulkLoader` implementation (`OneTimeBulkLoader`) to verify that `id.toString()` vertex lookups still work properly if no temporary property is written to the target graph.
This will not affect the default behavior of `BLVP`, since `IncrementalBulkLoader` will still be used by default, but it allowed to add some more test cases.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/d3ff7533
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/d3ff7533
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/d3ff7533
Branch: refs/heads/master
Commit: d3ff7533e05d210c5f6bc6ba76d0d2b4944dbde1
Parents: 8cc12c0
Author: Daniel Kuppitz <da...@hotmail.com>
Authored: Wed Dec 16 18:56:28 2015 +0100
Committer: Daniel Kuppitz <da...@hotmail.com>
Committed: Wed Dec 16 18:56:28 2015 +0100
----------------------------------------------------------------------
.../computer/bulkloading/BulkLoader.java | 6 +-
.../bulkloading/BulkLoaderVertexProgram.java | 10 +-
.../bulkloading/IncrementalBulkLoader.java | 4 +-
.../computer/bulkloading/OneTimeBulkLoader.java | 108 +++++++++++++++++++
.../process/GroovyProcessComputerSuite.java | 5 +-
.../BulkLoaderVertexProgramTest.java | 49 ++++++++-
6 files changed, 165 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d3ff7533/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoader.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoader.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoader.java
index dffe11e..fe1fb77 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoader.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoader.java
@@ -102,11 +102,7 @@ public interface BulkLoader {
* @param g A standard traversal source for the given graph.
* @return The matched vertex.
*/
- public default Vertex getVertex(final Vertex vertex, final Graph graph, final GraphTraversalSource g) {
- return useUserSuppliedIds()
- ? getVertexById(vertex.id(), graph, g)
- : g.V().has(getVertexIdProperty(), vertex.id()).next();
- }
+ public Vertex getVertex(final Vertex vertex, final Graph graph, final GraphTraversalSource g);
/**
* Gets a vertex by its ID from the given graph.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d3ff7533/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgram.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgram.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgram.java
index 2f3a0d0..3a10582 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgram.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgram.java
@@ -152,7 +152,7 @@ public class BulkLoaderVertexProgram implements VertexProgram<Tuple> {
ConfigurationUtils.copy(config, configuration);
}
intermediateBatchSize = configuration.getLong(INTERMEDIATE_BATCH_SIZE_CFG_KEY, 0L);
- elementComputeKeys.add(configuration.getString(BULK_LOADER_VERTEX_ID_CFG_KEY, DEFAULT_BULK_LOADER_VERTEX_ID));
+ elementComputeKeys.add(DEFAULT_BULK_LOADER_VERTEX_ID);
bulkLoader = createBulkLoader();
}
@@ -217,7 +217,7 @@ public class BulkLoaderVertexProgram implements VertexProgram<Tuple> {
this.commit(false);
if (!bulkLoader.useUserSuppliedIds()) {
// create an id pair and send it to all the vertex's incoming adjacent vertices
- sourceVertex.property(bulkLoader.getVertexIdProperty(), targetVertex.id().toString());
+ sourceVertex.property(DEFAULT_BULK_LOADER_VERTEX_ID, targetVertex.id());
messenger.sendMessage(messageScope, Pair.with(sourceVertex.id(), targetVertex.id()));
}
} else if (memory.getIteration() == 1) {
@@ -242,7 +242,7 @@ public class BulkLoaderVertexProgram implements VertexProgram<Tuple> {
idPairs.put(idPair.getValue(0), idPair.getValue(1));
}
// get the vertex with given the dummy id property
- final Object outVId = sourceVertex.value(bulkLoader.getVertexIdProperty());
+ final Object outVId = sourceVertex.value(DEFAULT_BULK_LOADER_VERTEX_ID);
final Vertex outV = bulkLoader.getVertexById(outVId, graph, g);
// for all the incoming edges of the vertex, get the incoming adjacent vertex and write the edge and its properties
sourceVertex.edges(Direction.OUT).forEachRemaining(edge -> {
@@ -253,7 +253,7 @@ public class BulkLoaderVertexProgram implements VertexProgram<Tuple> {
});
}
} else if (memory.getIteration() == 2) {
- final Object vertexId = sourceVertex.value(bulkLoader.getVertexIdProperty());
+ final Object vertexId = sourceVertex.value(DEFAULT_BULK_LOADER_VERTEX_ID);
bulkLoader.getVertexById(vertexId, graph, g)
.property(bulkLoader.getVertexIdProperty()).remove();
this.commit(false);
@@ -264,7 +264,7 @@ public class BulkLoaderVertexProgram implements VertexProgram<Tuple> {
public boolean terminate(final Memory memory) {
switch (memory.getIteration()) {
case 1:
- return bulkLoader.keepOriginalIds();
+ return bulkLoader.keepOriginalIds() || bulkLoader.getVertexIdProperty() == null;
case 2:
return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d3ff7533/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/IncrementalBulkLoader.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/IncrementalBulkLoader.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/IncrementalBulkLoader.java
index 187f72e..990334f 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/IncrementalBulkLoader.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/IncrementalBulkLoader.java
@@ -47,12 +47,12 @@ public class IncrementalBulkLoader implements BulkLoader {
public Vertex getOrCreateVertex(final Vertex vertex, final Graph graph, final GraphTraversalSource g) {
final Iterator<Vertex> iterator = useUserSuppliedIds()
? g.V().hasId(vertex.id())
- : g.V().has(vertex.label(), getVertexIdProperty(), vertex.id());
+ : g.V().has(vertex.label(), getVertexIdProperty(), vertex.id().toString());
return iterator.hasNext()
? iterator.next()
: useUserSuppliedIds()
? g.addV(vertex.label()).property(T.id, vertex.id()).next()
- : g.addV(vertex.label()).property(getVertexIdProperty(), vertex.id()).next();
+ : g.addV(vertex.label()).property(getVertexIdProperty(), vertex.id().toString()).next();
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d3ff7533/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/OneTimeBulkLoader.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/OneTimeBulkLoader.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/OneTimeBulkLoader.java
new file mode 100644
index 0000000..eb0bf1a
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/OneTimeBulkLoader.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.process.computer.bulkloading;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.Property;
+import org.apache.tinkerpop.gremlin.structure.T;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.VertexProperty;
+
+import java.util.Iterator;
+
+/**
+ * @author Daniel Kuppitz (http://gremlin.guru)
+ */
+public class OneTimeBulkLoader implements BulkLoader {
+
+ private boolean userSuppliedIds = false;
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public Vertex getOrCreateVertex(final Vertex vertex, final Graph graph, final GraphTraversalSource g) {
+ final GraphTraversal<Vertex, Vertex> t = g.addV(vertex.label());
+ return (useUserSuppliedIds() ? t.property(T.id, vertex.id()) : t).next();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public Edge getOrCreateEdge(final Edge edge, final Vertex outVertex, final Vertex inVertex, final Graph graph, final GraphTraversalSource g) {
+ return createEdge(edge, outVertex, inVertex, graph, g);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public VertexProperty getOrCreateVertexProperty(final VertexProperty<?> property, final Vertex vertex, final Graph graph, final GraphTraversalSource g) {
+ return createVertexProperty(property, vertex, graph, g);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public Vertex getVertex(final Vertex vertex, final Graph graph, final GraphTraversalSource g) {
+ return getVertexById(vertex.id(), graph, g);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean useUserSuppliedIds() {
+ return userSuppliedIds;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean keepOriginalIds() {
+ return false;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public String getVertexIdProperty() {
+ return null;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void configure(final Configuration configuration) {
+ if (configuration.containsKey(BulkLoaderVertexProgram.USER_SUPPLIED_IDS_CFG_KEY)) {
+ userSuppliedIds = configuration.getBoolean(BulkLoaderVertexProgram.USER_SUPPLIED_IDS_CFG_KEY);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d3ff7533/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java
----------------------------------------------------------------------
diff --git a/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java b/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java
index 6264fbc..56b6b34 100644
--- a/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java
+++ b/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java
@@ -24,7 +24,6 @@ import org.apache.tinkerpop.gremlin.groovy.loaders.SugarLoader;
import org.apache.tinkerpop.gremlin.groovy.util.SugarTestHelper;
import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
import org.apache.tinkerpop.gremlin.process.computer.bulkloading.BulkLoaderVertexProgramTest;
-import org.apache.tinkerpop.gremlin.process.computer.clustering.peerpressure.PeerPressureVertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.clustering.peerpressure.PeerPressureVertexProgramTest;
import org.apache.tinkerpop.gremlin.process.computer.ranking.pagerank.PageRankVertexProgramTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.branch.GroovyBranchTest;
@@ -104,7 +103,7 @@ public class GroovyProcessComputerSuite extends ProcessComputerSuite {
* as needed to enforce tests upon implementations.
*/
private static final Class<?>[] allTests = new Class<?>[]{
-/*
+
//branch
GroovyBranchTest.Traversals.class,
GroovyChooseTest.Traversals.class,
@@ -170,7 +169,7 @@ public class GroovyProcessComputerSuite extends ProcessComputerSuite {
// algorithms
PageRankVertexProgramTest.class,
- PeerPressureVertexProgramTest.class,*/
+ PeerPressureVertexProgramTest.class,
BulkLoaderVertexProgramTest.class,
};
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d3ff7533/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java
index aad15d4..a4d4d3a 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java
@@ -40,6 +40,7 @@ import org.junit.Test;
import java.io.File;
import java.lang.reflect.Field;
import java.util.Iterator;
+import java.util.concurrent.ExecutionException;
import java.util.function.Function;
import static org.apache.tinkerpop.gremlin.LoadGraphWith.GraphData.MODERN;
@@ -134,12 +135,56 @@ public class BulkLoaderVertexProgramTest extends AbstractGremlinProcessTest {
assertGraphEquality(graph, getWriteGraph());
}
+ @Test
+ @LoadGraphWith(MODERN)
+ public void shouldUseOneTimeBulkLoader() throws Exception {
+ for (int iteration = 1; iteration <= 2; iteration++) {
+ final BulkLoaderVertexProgram blvp = BulkLoaderVertexProgram.build()
+ .bulkLoader(OneTimeBulkLoader.class)
+ .writeGraph(getWriteGraphConfiguration()).create(graph);
+ final BulkLoader loader = getBulkLoader(blvp);
+ assertTrue(loader instanceof OneTimeBulkLoader);
+ graph.compute(graphComputerClass.get()).workers(1).program(blvp).submit().get();
+ final Graph result = getWriteGraph();
+ assertEquals(6 * iteration, IteratorUtils.count(result.vertices()));
+ assertEquals(6 * iteration, IteratorUtils.count(result.edges()));
+ result.close();
+ }
+ }
+
+ @Test
+ @LoadGraphWith(MODERN)
+ public void shouldUseOneTimeBulkLoaderWithUserSuppliedIds() throws Exception {
+ for (int iteration = 1; iteration <= 2; iteration++) {
+ final BulkLoaderVertexProgram blvp = BulkLoaderVertexProgram.build()
+ .bulkLoader(OneTimeBulkLoader.class)
+ .userSuppliedIds(true)
+ .writeGraph(getWriteGraphConfiguration()).create(graph);
+ final BulkLoader loader = getBulkLoader(blvp);
+ assertTrue(loader instanceof OneTimeBulkLoader);
+ try {
+ graph.compute(graphComputerClass.get()).workers(1).program(blvp).submit().get();
+ assertEquals(1, iteration);
+ final Graph result = getWriteGraph();
+ assertEquals(6, IteratorUtils.count(result.vertices()));
+ assertEquals(6, IteratorUtils.count(result.edges()));
+ result.close();
+ } catch (ExecutionException e) {
+ assertEquals(2, iteration);
+ Throwable cause = e.getCause();
+ while (cause.getCause() != null) {
+ cause = cause.getCause();
+ }
+ assertTrue(cause.getMessage().startsWith("Vertex with id already exists"));
+ }
+ }
+ }
+
private static void assertGraphEquality(final Graph source, final Graph target) {
assertGraphEquality(source, target, Element::id);
}
private static void assertGraphEquality(final Graph source, final Graph target, final Function<Vertex, Object> idAccessor) {
- final GraphTraversalSource sg = source.traversal();
final GraphTraversalSource tg = target.traversal();
assertEquals(IteratorUtils.count(source.vertices()), IteratorUtils.count(target.vertices()));
assertEquals(IteratorUtils.count(target.edges()), IteratorUtils.count(target.edges()));
@@ -148,7 +193,7 @@ public class BulkLoaderVertexProgramTest extends AbstractGremlinProcessTest {
final Iterator<Vertex> vertexIterator = target.vertices();
while (vertexIterator.hasNext()) {
final Vertex v = vertexIterator.next();
- if (idAccessor.apply(v).equals(originalVertex.id())) {
+ if (idAccessor.apply(v).toString().equals(originalVertex.id().toString())) {
tmpVertex = v;
break;
}