You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by dk...@apache.org on 2015/08/31 20:39:08 UTC
incubator-tinkerpop git commit: tweaked BLVP configuration
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/blvp e5d519234 -> b1191e743
tweaked BLVP configuration
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/b1191e74
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/b1191e74
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/b1191e74
Branch: refs/heads/blvp
Commit: b1191e743a2b574a7bcc990ad27d99ab9df21f8f
Parents: e5d5192
Author: Daniel Kuppitz <da...@hotmail.com>
Authored: Mon Aug 31 20:38:27 2015 +0200
Committer: Daniel Kuppitz <da...@hotmail.com>
Committed: Mon Aug 31 20:38:27 2015 +0200
----------------------------------------------------------------------
.../gremlin/process/computer/VertexProgram.java | 4 +
.../bulkloading/BulkLoaderVertexProgram.java | 83 +++++++++++++++++---
.../bulkloading/IncrementalBulkLoader.java | 11 +--
.../util/AbstractVertexProgramBuilder.java | 5 +-
.../BulkLoaderVertexProgramTest.java | 17 +---
5 files changed, 90 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b1191e74/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/VertexProgram.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/VertexProgram.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/VertexProgram.java
index 4fd9e82..37ff8e7 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/VertexProgram.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/VertexProgram.java
@@ -223,6 +223,10 @@ public interface VertexProgram<M> extends Cloneable {
public interface Builder {
+ /**
+ * This method should only be used by the underlying compute engine. For VertexProgram configurations, please
+ * use specific fluent methods off the builder.
+ */
public Builder configure(final Object... keyValues);
public <P extends VertexProgram> P create(final Graph graph);
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b1191e74/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgram.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgram.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgram.java
index 9ab21c2..eaebd53 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgram.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgram.java
@@ -20,7 +20,9 @@ package org.apache.tinkerpop.gremlin.process.computer.bulkloading;
import org.apache.commons.configuration.BaseConfiguration;
import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.ConfigurationUtils;
+import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
import org.apache.tinkerpop.gremlin.process.computer.Memory;
import org.apache.tinkerpop.gremlin.process.computer.MessageScope;
@@ -58,6 +60,8 @@ public class BulkLoaderVertexProgram implements VertexProgram<Tuple> {
public static final String BULK_LOADER_VERTEX_PROGRAM_CFG_PREFIX = "gremlin.bulkLoaderVertexProgram";
public static final String GRAPH_CFG_KEY = "graph";
public static final String BULK_LOADER_CFG_KEY = "loader";
+ public final static String USER_SUPPLIED_IDS_CFG_KEY = "userSuppliedIds";
+ public final static String KEEP_ORIGINAL_IDS_CFG_KEY = "keepOriginalIds";
public static final String INTERMEDIATE_BATCH_SIZE_CFG_KEY = "intermediateBatchSize";
public static final String BULK_LOADER_VERTEX_ID_CFG_KEY = "vertexIdProperty";
public static final String DEFAULT_BULK_LOADER_VERTEX_ID = "bulkLoader.vertex.id";
@@ -81,13 +85,6 @@ public class BulkLoaderVertexProgram implements VertexProgram<Tuple> {
elementComputeKeys = new HashSet<>();
}
- private Configuration getGraphConfiguration() {
- final Configuration config = configuration.subset(GRAPH_CFG_KEY);
- config.setProperty(Graph.GRAPH, config.getString("class"));
- config.clearProperty("class");
- return config;
- }
-
private BulkLoader createBulkLoader() {
final BulkLoader loader;
final Configuration config = configuration.subset(BULK_LOADER_CFG_KEY);
@@ -118,7 +115,7 @@ public class BulkLoaderVertexProgram implements VertexProgram<Tuple> {
* @param close Whether to close the current graph instance after calling commit() or not.
*/
private void commit(final boolean close) {
- if (!close && (counter.get().incrementAndGet() % intermediateBatchSize != 0 || intermediateBatchSize == 0L))
+ if (!close && (intermediateBatchSize == 0L || counter.get().incrementAndGet() % intermediateBatchSize != 0))
return;
if (null != graph) {
if (graph.features().graph().supportsTransactions()) {
@@ -176,7 +173,7 @@ public class BulkLoaderVertexProgram implements VertexProgram<Tuple> {
@Override
public void workerIterationStart(final Memory memory) {
if (null == graph) {
- graph = GraphFactory.open(getGraphConfiguration());
+ graph = GraphFactory.open(configuration.subset(GRAPH_CFG_KEY));
LOGGER.info("Opened Graph instance: {}", graph);
try {
if (!graph.features().graph().supportsConcurrentAccess()) {
@@ -321,5 +318,73 @@ public class BulkLoaderVertexProgram implements VertexProgram<Tuple> {
ConfigurationUtils.append(graph.configuration().subset(BULK_LOADER_VERTEX_PROGRAM_CFG_PREFIX), configuration);
return (BulkLoaderVertexProgram) VertexProgram.createVertexProgram(graph, this.configuration);
}
+
+ private void setLoaderConfigurationProperty(final String key, final Object value) {
+ configuration.setProperty(String.join(".", BULK_LOADER_CFG_KEY, key), value);
+ }
+
+ private void setGraphConfigurationProperty(final String key, final Object value) {
+ configuration.setProperty(String.join(".", GRAPH_CFG_KEY, key), value);
+ }
+
+ /**
+ * Sets the class name of the BulkLoader implementation to be used.
+ */
+ public Builder bulkLoader(final String className) {
+ setLoaderConfigurationProperty(Graph.GRAPH, className);
+ return this;
+ }
+
+ /**
+ * Sets the class of the BulkLoader implementation to be used.
+ */
+ public Builder bulkLoader(final Class<? extends BulkLoader> clazz) {
+ return bulkLoader(clazz.getCanonicalName());
+ }
+
+ /**
+ * Sets the name of the property that is used to store the original vertex identifiers in the target graph.
+ */
+ public Builder vertexIdProperty(final String name) {
+ setLoaderConfigurationProperty(BULK_LOADER_VERTEX_ID_CFG_KEY, name);
+ return this;
+ }
+
+ /**
+ * Specifies whether user supplied identifiers should be used when the bulk loader creates vertices in the
+ * target graph.
+ */
+ public Builder userSuppliedIds(final boolean useUserSuppliedIds) {
+ setLoaderConfigurationProperty(USER_SUPPLIED_IDS_CFG_KEY, useUserSuppliedIds);
+ return this;
+ }
+
+ /**
+ * Specifies whether the original vertex identifiers should be kept in the target graph or not. In case of false
+ * BulkLoaderVertexProgram will add another iteration to remove the properties and it won't be possible to use
+ * the data for further incremental bulk loads.
+ */
+ public Builder keepOriginalIds(final boolean keepOriginalIds) {
+ setLoaderConfigurationProperty(KEEP_ORIGINAL_IDS_CFG_KEY, keepOriginalIds);
+ return this;
+ }
+
+ /**
+ * The batch size for a single transaction (number of vertices in the vertex loading stage; number of edges in
+ * the edge loading stage).
+ */
+ public Builder intermediateBatchSize(final int batchSize) {
+ configuration.setProperty(INTERMEDIATE_BATCH_SIZE_CFG_KEY, batchSize);
+ return this;
+ }
+
+ /**
+ * A configuration for the target graph that can be passed to GraphFactory.open().
+ */
+ public Builder writeGraph(final String configurationFile) throws ConfigurationException {
+ final Configuration conf = new PropertiesConfiguration(configurationFile);
+ conf.getKeys().forEachRemaining(key -> setGraphConfigurationProperty(key, conf.getProperty(key)));
+ return this;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b1191e74/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/IncrementalBulkLoader.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/IncrementalBulkLoader.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/IncrementalBulkLoader.java
index 2177d0b..f03cd18 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/IncrementalBulkLoader.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/IncrementalBulkLoader.java
@@ -36,9 +36,6 @@ import java.util.Iterator;
*/
public class IncrementalBulkLoader implements BulkLoader {
- public final static String USER_SUPPLIED_IDS_CFG_KEY = "userSuppliedIds";
- public final static String KEEP_ORIGINAL_IDS_CFG_KEY = "keepOriginalIds";
-
private String bulkLoaderVertexId = BulkLoaderVertexProgram.DEFAULT_BULK_LOADER_VERTEX_ID;
private boolean keepOriginalIds = true;
private boolean userSuppliedIds = false;
@@ -143,11 +140,11 @@ public class IncrementalBulkLoader implements BulkLoader {
if (configuration.containsKey(BulkLoaderVertexProgram.BULK_LOADER_VERTEX_ID_CFG_KEY)) {
bulkLoaderVertexId = configuration.getString(BulkLoaderVertexProgram.BULK_LOADER_VERTEX_ID_CFG_KEY);
}
- if (configuration.containsKey(USER_SUPPLIED_IDS_CFG_KEY)) {
- userSuppliedIds = configuration.getBoolean(USER_SUPPLIED_IDS_CFG_KEY);
+ if (configuration.containsKey(BulkLoaderVertexProgram.USER_SUPPLIED_IDS_CFG_KEY)) {
+ userSuppliedIds = configuration.getBoolean(BulkLoaderVertexProgram.USER_SUPPLIED_IDS_CFG_KEY);
}
- if (configuration.containsKey(KEEP_ORIGINAL_IDS_CFG_KEY)) {
- keepOriginalIds = configuration.getBoolean(KEEP_ORIGINAL_IDS_CFG_KEY);
+ if (configuration.containsKey(BulkLoaderVertexProgram.KEEP_ORIGINAL_IDS_CFG_KEY)) {
+ keepOriginalIds = configuration.getBoolean(BulkLoaderVertexProgram.KEEP_ORIGINAL_IDS_CFG_KEY);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b1191e74/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/AbstractVertexProgramBuilder.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/AbstractVertexProgramBuilder.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/AbstractVertexProgramBuilder.java
index 035b685..50e5157 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/AbstractVertexProgramBuilder.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/AbstractVertexProgramBuilder.java
@@ -18,8 +18,8 @@
*/
package org.apache.tinkerpop.gremlin.process.computer.util;
-import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
import org.apache.commons.configuration.BaseConfiguration;
+import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
import org.apache.tinkerpop.gremlin.structure.Graph;
/**
@@ -44,6 +44,9 @@ public abstract class AbstractVertexProgramBuilder<B extends VertexProgram.Build
return (B) this;
}*/
+ /**
+ * {@inheritDoc}
+ */
@Override
public B configure(final Object... keyValues) {
VertexProgramHelper.legalConfigurationKeyValueArray(keyValues);
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b1191e74/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java
index 3584c00..d3e42f7 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java
@@ -22,7 +22,6 @@ import org.apache.commons.configuration.BaseConfiguration;
import org.apache.commons.configuration.Configuration;
import org.apache.tinkerpop.gremlin.LoadGraphWith;
import org.apache.tinkerpop.gremlin.process.AbstractGremlinProcessTest;
-import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
import org.junit.Test;
@@ -38,11 +37,7 @@ import static org.junit.Assert.assertTrue;
*/
public class BulkLoaderVertexProgramTest extends AbstractGremlinProcessTest {
- private BulkLoader getBulkLoader(final Configuration configuration) throws Exception {
- final VertexProgram.Builder builder = BulkLoaderVertexProgram.build();
- if (configuration != null)
- configuration.getKeys().forEachRemaining(key -> builder.configure(key, configuration.getProperty(key)));
- BulkLoaderVertexProgram blvp = builder.create(graph);
+ private BulkLoader getBulkLoader(final BulkLoaderVertexProgram blvp) throws Exception {
final Field field = BulkLoaderVertexProgram.class.getDeclaredField("bulkLoader");
field.setAccessible(true);
return (BulkLoader) field.get(blvp);
@@ -56,7 +51,7 @@ public class BulkLoaderVertexProgramTest extends AbstractGremlinProcessTest {
@Test
public void shouldUseIncrementalBulkLoaderByDefault() throws Exception {
- final BulkLoader loader = getBulkLoader(null);
+ final BulkLoader loader = getBulkLoader(BulkLoaderVertexProgram.build().create(graph));
assertTrue(loader instanceof IncrementalBulkLoader);
assertTrue(loader.keepOriginalIds());
assertFalse(loader.useUserSuppliedIds());
@@ -65,9 +60,7 @@ public class BulkLoaderVertexProgramTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(MODERN)
public void shouldStoreOriginalIds() throws Exception {
- final Configuration configuration = new BaseConfiguration();
- configuration.setProperty("loader.userSuppliedIds", false);
- final BulkLoader loader = getBulkLoader(configuration);
+ final BulkLoader loader = getBulkLoader(BulkLoaderVertexProgram.build().userSuppliedIds(false).create(graph));
assertFalse(loader.useUserSuppliedIds());
final Graph target = getTargetGraph();
graph.vertices().forEachRemaining(v -> loader.getOrCreateVertex(v, target, target.traversal()));
@@ -77,9 +70,7 @@ public class BulkLoaderVertexProgramTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(MODERN)
public void shouldNotStoreOriginalIds() throws Exception {
- final Configuration configuration = new BaseConfiguration();
- configuration.setProperty("loader.userSuppliedIds", true);
- final BulkLoader loader = getBulkLoader(configuration);
+ final BulkLoader loader = getBulkLoader(BulkLoaderVertexProgram.build().userSuppliedIds(true).create(graph));
assertTrue(loader.useUserSuppliedIds());
final Graph target = getTargetGraph();
graph.vertices().forEachRemaining(v -> loader.getOrCreateVertex(v, target, target.traversal()));