You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2016/12/21 18:57:53 UTC
tinkerpop git commit: Tweaks to SparkGraphComputer configuration
settings. More GraphComputerTest things. Before the break,
I want to have Configuration all stubbed out for GraphComputer and GraphActors
and then this branch is simply a function of testin
Repository: tinkerpop
Updated Branches:
refs/heads/TINKERPOP-1564 6639e4cd7 -> cbc484525
Tweaks to SparkGraphComputer configuration settings. More GraphComputerTest things. Before the break, I want to have Configuration all stubbed out for GraphComputer and GraphActors and then this branch is simply a function of testing GraphActors at scale when we return.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/cbc48452
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/cbc48452
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/cbc48452
Branch: refs/heads/TINKERPOP-1564
Commit: cbc484525f2af99eeeffcefa6d98f5905c4115cf
Parents: 6639e4c
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed Dec 21 11:57:48 2016 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed Dec 21 11:57:48 2016 -0700
----------------------------------------------------------------------
.../gremlin/process/computer/GraphComputer.java | 4 ++++
.../process/computer/GraphComputerTest.java | 24 ++++++++------------
.../computer/AbstractHadoopGraphComputer.java | 6 ++---
.../process/computer/SparkGraphComputer.java | 5 ++--
.../process/computer/TinkerGraphComputer.java | 4 +++-
5 files changed, 22 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/cbc48452/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputer.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputer.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputer.java
index 25074fd..14ab0ec 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputer.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputer.java
@@ -275,6 +275,10 @@ public interface GraphComputer extends Processor {
private Exceptions() {
}
+ public static UnsupportedOperationException graphNotSupported(final Graph graph) {
+ return new UnsupportedOperationException("The provided graph is not supported: " + graph.getClass().getCanonicalName());
+ }
+
public static UnsupportedOperationException adjacentVertexLabelsCanNotBeRead() {
return new UnsupportedOperationException("The label of an adjacent vertex can not be read");
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/cbc48452/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java
index a166423..9a99d0a 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java
@@ -138,30 +138,26 @@ public class GraphComputerTest extends AbstractGremlinProcessTest {
final Configuration graphConfiguration = new BaseConfiguration();
ConfigurationUtils.copy(graph.configuration(), graphConfiguration);
final GraphComputer graphComputer = graphProvider.getGraphComputer(graph);
- final Configuration graphComputerConfiguration = graphComputer.configuration();
- final Configuration graphComputerConfigurationClone = new BaseConfiguration();
- ConfigurationUtils.copy(graphComputerConfiguration, graphComputerConfigurationClone);
- assertEquals(ConfigurationConverter.getMap(graphComputerConfiguration), ConfigurationConverter.getMap(graphComputerConfigurationClone));
// creating a graph computer shouldn't alter the graph configuration
assertEquals(ConfigurationConverter.getMap(graphConfiguration), ConfigurationConverter.getMap(graph.configuration()));
// creating a graph computer should add the graph computer's class name
- assertTrue(graphComputerConfiguration.containsKey(GraphComputer.GRAPH_COMPUTER));
- assertEquals(graphComputer.getClass().getCanonicalName(), graphComputerConfiguration.getString(GraphComputer.GRAPH_COMPUTER));
+ assertTrue(graphComputer.configuration().containsKey(GraphComputer.GRAPH_COMPUTER));
+ assertEquals(graphComputer.getClass().getCanonicalName(), graphComputer.configuration().getString(GraphComputer.GRAPH_COMPUTER));
// specifying worker count should alter the graph computer configuration
- int workers = graphComputerConfiguration.containsKey(GraphComputer.WORKERS) ? graphComputerConfiguration.getInt(GraphComputer.WORKERS) : 1;
+ int workers = graphComputer.configuration().getInt(GraphComputer.WORKERS,1);
graphComputer.workers(workers + 1);
- assertTrue(graphComputerConfiguration.containsKey(GraphComputer.WORKERS));
- assertEquals(graphComputerConfiguration.getInt(GraphComputer.WORKERS), workers + 1);
- assertEquals(ConfigurationConverter.getMap(graphComputerConfiguration), ConfigurationConverter.getMap(graphComputer.configuration()));
- graphComputerConfigurationClone.clear();
- ConfigurationUtils.copy(graphComputer.configuration(), graphComputerConfigurationClone);
- assertEquals(ConfigurationConverter.getMap(graphComputerConfiguration), ConfigurationConverter.getMap(graphComputerConfigurationClone));
+ assertTrue(graphComputer.configuration().containsKey(GraphComputer.WORKERS));
+ assertEquals(graphComputer.configuration().getInt(GraphComputer.WORKERS), workers + 1);
+ // make a copy of configuration before executing graph computer
+ final Configuration tempConfiguration = new BaseConfiguration();
+ ConfigurationUtils.copy(graphComputer.configuration(),tempConfiguration);
// execute graph computer
graphComputer.program(new VertexProgramG()).submit(graph);
// executing a graph computer should not alter the graph configuration
assertEquals(ConfigurationConverter.getMap(graphConfiguration), ConfigurationConverter.getMap(graph.configuration()));
// executing a graph computer should not alter the graph computer configuration
- // TODO: assertEquals(ConfigurationConverter.getMap(graphComputerConfiguration), ConfigurationConverter.getMap(graphComputerConfigurationClone));
+ assertEquals(ConfigurationConverter.getMap(tempConfiguration), ConfigurationConverter.getMap(graphComputer.configuration()));
+ assertEquals(ConfigurationConverter.getMap(graphConfiguration), ConfigurationConverter.getMap(graph.configuration()));
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/cbc48452/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
index a81efb6..b607446 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java
@@ -80,10 +80,10 @@ public abstract class AbstractHadoopGraphComputer implements GraphComputer {
}
protected AbstractHadoopGraphComputer(final org.apache.commons.configuration.Configuration configuration) {
- this.configuration = new HadoopConfiguration(configuration);
+ this.configuration = new HadoopConfiguration();
this.configuration.setProperty(GRAPH_COMPUTER, this.getClass().getCanonicalName());
+ GraphComputerHelper.configure(this, configuration);
this.logger = LoggerFactory.getLogger(this.getClass());
- GraphComputerHelper.configure(this, this.configuration);
}
@Override
@@ -135,7 +135,7 @@ public abstract class AbstractHadoopGraphComputer implements GraphComputer {
@Override
public Future<ComputerResult> submit(final Graph graph) {
- ConfigurationUtils.copy(graph.configuration(), this.configuration);
+ GraphComputerHelper.configure(this,graph.configuration());
return this.submit();
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/cbc48452/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
index c72454b..cf15e5e 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
@@ -18,7 +18,6 @@
*/
package org.apache.tinkerpop.gremlin.spark.process.computer;
-import org.apache.commons.configuration.ConfigurationUtils;
import org.apache.commons.configuration.FileConfiguration;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.lang3.concurrent.BasicThreadFactory;
@@ -51,7 +50,6 @@ import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
import org.apache.tinkerpop.gremlin.process.computer.Memory;
import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.util.DefaultComputerResult;
-import org.apache.tinkerpop.gremlin.process.computer.util.GraphComputerHelper;
import org.apache.tinkerpop.gremlin.process.computer.util.MapMemory;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategies;
import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalInterruptedException;
@@ -70,7 +68,6 @@ import org.apache.tinkerpop.gremlin.spark.structure.io.PersistedOutputRDD;
import org.apache.tinkerpop.gremlin.spark.structure.io.SparkContextStorage;
import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
import org.apache.tinkerpop.gremlin.structure.Direction;
-import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.io.Storage;
import java.io.File;
@@ -122,6 +119,8 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
@Override
public GraphComputer configure(final String key, final Object value) {
+ if (key.equals(SparkLauncher.SPARK_MASTER) && value.toString().startsWith("local"))
+ this.workers(Integer.valueOf(value.toString().substring(6).replace("]", "")));
this.configuration.setProperty(key, value);
return this;
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/cbc48452/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java
index e4ca684..9370895 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java
@@ -96,7 +96,7 @@ public final class TinkerGraphComputer implements GraphComputer {
private TinkerGraphComputer(final Configuration configuration) {
this.graph = null;
this.configuration = new BaseConfiguration();
- ConfigurationUtils.copy(configuration,this.configuration);
+ ConfigurationUtils.copy(configuration, this.configuration);
this.configuration.setProperty(GRAPH_COMPUTER, TinkerGraphComputer.class.getCanonicalName());
GraphComputerHelper.configure(this, configuration);
}
@@ -164,6 +164,8 @@ public final class TinkerGraphComputer implements GraphComputer {
@Override
public Future<ComputerResult> submit(final Graph graph) {
+ if (!(graph instanceof TinkerGraph))
+ throw GraphComputer.Exceptions.graphNotSupported(graph);
this.graph = (TinkerGraph) graph;
return this.submit();
}